Improved Server tests
diff --git a/pom.xml b/pom.xml
index 8a3a06f..faefc7b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -177,6 +177,17 @@
<version>1.13-b01</version>
</dependency>
+ <!-- Multipart support for Jersey -->
+ <!--
+ See
+ http://www.mkyong.com/webservices/jax-rs/file-upload-example-in-jersey/
+ <dependency>
+ <groupId>com.sun.jersey.contribs</groupId>
+ <artifactId>jersey-multipart</artifactId>
+ <version>1.13</version>
+ </dependency>
+ -->
+
<!-- JSON support -->
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
diff --git a/src/main/java/de/ids_mannheim/korap/KorapDocument.java b/src/main/java/de/ids_mannheim/korap/KorapDocument.java
index 349132e..3857ecc 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapDocument.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapDocument.java
@@ -21,7 +21,7 @@
private KorapPrimaryData primaryData;
@JsonIgnore
- public int internalDocID, localDocID;
+ public int internalDocID, localDocID, UID;
private String author, textClass, corpusID,
pubPlace, ID, title, subTitle,
@@ -106,6 +106,15 @@
this.ID = ID;
};
+ public void setUID (int UID) {
+ this.UID = UID;
+ };
+
+ @JsonProperty("UID")
+ public int getUID () {
+ return this.UID;
+ };
+
@JsonProperty("ID")
public String getID () {
return this.ID;
diff --git a/src/main/java/de/ids_mannheim/korap/KorapIndex.java b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
index b22f2fc..cbcbd1a 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
@@ -294,6 +294,13 @@
return fd;
};
+ // Add document to index as JSON object with a unique ID
+ public FieldDocument addDoc (int uid, String json) throws IOException {
+ FieldDocument fd = this.mapper.readValue(json, FieldDocument.class);
+ fd.setUID(uid);
+ return this.addDoc(fd);
+ };
+
// Add document to index as JSON object
public FieldDocument addDoc (String json) throws IOException {
@@ -353,6 +360,10 @@
};
};
+ // Return the number of unstaged texts
+ public int getUnstaged () {
+ return this.commitCounter;
+ };
// Get autoCommit valiue
public int autoCommit () {
diff --git a/src/main/java/de/ids_mannheim/korap/KorapNode.java b/src/main/java/de/ids_mannheim/korap/KorapNode.java
index e50a990..0c88f75 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapNode.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapNode.java
@@ -32,10 +32,13 @@
private static KorapIndex index;
/*
- Todo: Use korap.config for paths to
- indexDirectory
+ Todo: Use korap.config for paths to indexDirectory
*/
- private static String path = new String("/home/ndiewald/Repositories/korap/KorAP-modules/KorAP-lucene-index/sandbox/index");
+ private static String path =
+ new String("/home/ndiewald/Repositories/korap/KorAP-modules/KorAP-lucene-index/sandbox/index");
+
+ private static String name = "tanja";
+
/**
* Starts Grizzly HTTP server exposing JAX-RS resources defined in this application.
@@ -51,7 +54,24 @@
// create and start a new instance of grizzly http server
// exposing the Jersey application at BASE_URI
return GrizzlyHttpServerFactory.createHttpServer(URI.create(BASE_URI), rc);
- }
+ };
+
+
+ public static HttpServer startServer(String nodeName, String indexPath) {
+
+ // create a resource config that scans for JAX-RS resources and providers
+ // in de.ids_mannheim.korap.server package
+ final ResourceConfig rc =
+ new ResourceConfig().packages("de.ids_mannheim.korap.server");
+
+ name = nodeName;
+ path = indexPath;
+
+ // create and start a new instance of grizzly http server
+ // exposing the Jersey application at BASE_URI
+ return GrizzlyHttpServerFactory.createHttpServer(URI.create(BASE_URI), rc);
+ };
+
/**
* Main method.
@@ -66,30 +86,36 @@
server.stop();
};
+ public static String getName () {
+ return name;
+ };
+
+ // Get Index
public static KorapIndex getIndex () {
if (index != null)
return index;
try {
- File file = new File(path);
+ if (path == null) {
+ // Temporary index
+ index = new KorapIndex();
+ }
- log.info("Loading index from {}", path);
- if (!file.exists()) {
- log.error("Index not found at {}", path);
- return null;
+ else {
+ File file = new File(path);
+
+ log.info("Loading index from {}", path);
+ if (!file.exists()) {
+ log.error("Index not found at {}", path);
+ return null;
+ };
+
+ System.out.println("Loading index from " + path);
+
+ // Real index
+ index = new KorapIndex(new MMapDirectory(file));
};
-
- System.out.println("Loading index from " + path);
-
- index = new KorapIndex(new MMapDirectory(file));
return index;
- /*
- // Temporarily!
- static String path = new String();
-
- this.index = new KorapIndex(new MMapDirectory(f));
- return this.index;
- */
}
catch (IOException e) {
log.error("Index not loadable at {}: {}", path, e.getMessage());
diff --git a/src/main/java/de/ids_mannheim/korap/index/FieldDocument.java b/src/main/java/de/ids_mannheim/korap/index/FieldDocument.java
index 93edca3..e7ad688 100644
--- a/src/main/java/de/ids_mannheim/korap/index/FieldDocument.java
+++ b/src/main/java/de/ids_mannheim/korap/index/FieldDocument.java
@@ -20,8 +20,8 @@
import java.util.*;
/*
-Todo: Store primary data at base/cons field.
-All other Termvectors should have no stored field!
+ Todo: Store primary data at base/cons field.
+ All other Termvectors should have no stored field!
*/
/**
@@ -234,6 +234,12 @@
};
@Override
+ public void setUID (int ID) {
+ super.setUID(ID);
+ this.addInt("UID", ID);
+ };
+
+ @Override
public void setLayerInfo (String layerInfo) {
System.err.println(layerInfo);
super.setLayerInfo(layerInfo);
diff --git a/src/main/java/de/ids_mannheim/korap/query/SpanWithinQuery.java b/src/main/java/de/ids_mannheim/korap/query/SpanWithinQuery.java
index a314475..e0d23cd 100644
--- a/src/main/java/de/ids_mannheim/korap/query/SpanWithinQuery.java
+++ b/src/main/java/de/ids_mannheim/korap/query/SpanWithinQuery.java
@@ -164,27 +164,27 @@
/*
* Rewrite query in case it includes regular expressions or wildcards
*/
-
@Override
public Query rewrite (IndexReader reader) throws IOException {
SpanWithinQuery clone = null;
+ // Does the embedded query needs a rewrite?
SpanQuery query = (SpanQuery) embedded.rewrite(reader);
-
if (query != embedded) {
if (clone == null)
clone = this.clone();
clone.embedded = query;
};
+ // Does the wrap query needs a rewrite?
query = (SpanQuery) wrap.rewrite(reader);
-
if (query != wrap) {
if (clone == null)
clone = this.clone();
clone.wrap = query;
};
+ // There is a clone and it is important
if (clone != null)
return clone;
diff --git a/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java b/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
new file mode 100644
index 0000000..63e55bd
--- /dev/null
+++ b/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
@@ -0,0 +1,100 @@
+package de.ids_mannheim.korap.server;
+import java.util.*;
+import java.io.*;
+
+import com.fasterxml.jackson.annotation.*;
+import com.fasterxml.jackson.annotation.JsonInclude.Include;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.SerializationFeature;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/*
+ Todo: Ignore unstaged information as this may be incorrect in
+ Multithreading environment.
+*/
+
+@JsonInclude(Include.NON_NULL)
+public class KorapResponse {
+ ObjectMapper mapper = new ObjectMapper();
+
+ private String errstr, msg, version, node;
+ private int err, unstaged = 0;
+
+ public KorapResponse (String node, String version) {
+ this.setNode(node);
+ this.setVersion(version);
+ };
+
+ public KorapResponse () {};
+
+ public KorapResponse setErrstr (String msg) {
+ this.errstr = msg;
+ return this;
+ };
+
+ public String getErrstr () {
+ return this.errstr;
+ };
+
+ public KorapResponse setErr (int num) {
+ this.err = num;
+ return this;
+ };
+
+ public int getErr () {
+ return this.err;
+ };
+
+ public KorapResponse setMsg (String msg) {
+ this.msg = msg;
+ return this;
+ };
+
+ public String getMsg () {
+ return this.msg;
+ };
+
+ public String getVersion () {
+ return this.version;
+ };
+
+ public KorapResponse setVersion (String version) {
+ this.version = version;
+ return this;
+ };
+
+ public String getNode () {
+ return this.node;
+ };
+
+ public KorapResponse setNode (String name) {
+ this.node = name;
+ return this;
+ };
+
+ public int getUnstaged () {
+ return this.unstaged;
+ };
+
+ public KorapResponse setUnstaged (int unstaged) {
+ this.unstaged = unstaged;
+ return this;
+ };
+
+ // Serialize
+ public String toJSON () {
+ ObjectNode json = (ObjectNode) mapper.valueToTree(this);
+ if (json.size() == 0)
+ return "{}";
+
+ try {
+ return mapper.writeValueAsString(json);
+ }
+ catch (Exception e) {
+ this.errstr = e.getLocalizedMessage();
+ };
+
+ return "{\"errstr\" : \"" + this.errstr + "\"}";
+ };
+};
diff --git a/src/main/java/de/ids_mannheim/korap/server/Resource.java b/src/main/java/de/ids_mannheim/korap/server/Resource.java
index e500a81..2ac540a 100644
--- a/src/main/java/de/ids_mannheim/korap/server/Resource.java
+++ b/src/main/java/de/ids_mannheim/korap/server/Resource.java
@@ -1,7 +1,10 @@
package de.ids_mannheim.korap.server;
+import java.io.*;
+
import javax.ws.rs.GET;
import javax.ws.rs.POST;
+import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.Produces;
@@ -11,20 +14,34 @@
import javax.ws.rs.core.Context;
import javax.ws.rs.core.UriInfo;
import javax.ws.rs.core.MultivaluedMap;
+import javax.ws.rs.ext.ReaderInterceptor;
+import javax.ws.rs.ext.ReaderInterceptorContext;
+import javax.ws.rs.ext.WriterInterceptor;
+import javax.ws.rs.ext.WriterInterceptorContext;
+import javax.ws.rs.WebApplicationException;
import de.ids_mannheim.korap.KorapNode;
import de.ids_mannheim.korap.KorapIndex;
import de.ids_mannheim.korap.KorapSearch;
import de.ids_mannheim.korap.KorapMatch;
import de.ids_mannheim.korap.KorapResult;
+import de.ids_mannheim.korap.server.KorapResponse;
+import de.ids_mannheim.korap.index.FieldDocument;
import de.ids_mannheim.korap.util.QueryException;
import java.util.List;
import java.util.regex.Pattern;
import java.util.regex.Matcher;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
import com.mchange.v2.c3p0.*;
+/*
+ http://www.vogella.com/tutorials/REST/article.html
+ Write a simple server response class!
+*/
+
/**
* Root resource (exposed at root path)
*
@@ -36,6 +53,8 @@
public class Resource {
static Pattern p = Pattern.compile("\\s*(?i:false|null)\\s*");
+ private String version;
+
private static boolean isNull (String value) {
if (value == null)
@@ -49,6 +68,92 @@
};
/**
+ * Add new documents to the index
+ *
+ * @param json JSON-LD string with search and potential meta filters.
+ */
+ /*
+ * Support GZip:
+ * http://stackoverflow.com/questions/19765582/how-to-make-jersey-use-gzip-compression-for-the-response-message-body
+ */
+ @PUT
+ @Path("/{textID}")
+ @Produces(MediaType.APPLICATION_JSON)
+ @Consumes(MediaType.APPLICATION_JSON)
+ public String add (@PathParam("textID") Integer uid,
+ @Context UriInfo uri,
+ String json) {
+ /*
+ * See
+ * http://www.mkyong.com/webservices/jax-rs/file-upload-example-in-jersey/
+ */
+
+ // Todo: Parameter for server node
+
+ // Get index
+ KorapIndex index = KorapNode.getIndex();
+ KorapResponse kresp = new KorapResponse(KorapNode.getName(), index.getVersion());
+
+ // kresp.setErr();
+ if (index == null)
+ return kresp.setErrstr("Unable to find index").toJSON();
+
+ String ID = "Unknown";
+ int unstaged = 0;
+ try {
+ FieldDocument fd = index.addDoc(uid, json);
+ ID = fd.getID();
+ unstaged = index.getUnstaged();
+ }
+ // Set HTTP to ???
+ catch (IOException e) {
+
+ return kresp.setErrstr(e.getLocalizedMessage()).toJSON();
+ };
+
+ // Set HTTP to 200
+ return kresp.
+ setMsg("Text \"" + ID + "\" was added successfully")
+ .setUnstaged(unstaged)
+ .toJSON();
+ };
+
+
+ // TODO: Commit changes to the index before the server dies!
+ /**
+ * Commit data changes to the index
+ */
+ @POST
+ @Produces(MediaType.APPLICATION_JSON)
+ public String commit () {
+
+ // Get index
+ KorapIndex index = KorapNode.getIndex();
+ KorapResponse kresp = new KorapResponse(KorapNode.getName(), index.getVersion());
+
+ if (version == null)
+ version = index.getVersion();
+
+ // There are documents to commit
+ if (index.getUnstaged() != 0) {
+ try {
+ index.commit();
+ }
+ catch (IOException e) {
+ // Set HTTP to ???
+ return kresp.setErrstr(e.getMessage()).toJSON();
+ };
+
+ // Set HTTP to ???
+ return kresp.setMsg("Unstaged data was committed").toJSON();
+ };
+
+ // Set HTTP to ???
+ return kresp.setMsg("No unstaged data available").toJSON();
+ };
+
+
+ /**
* Search the lucene index.
*
* @param json JSON-LD string with search and potential meta filters.
@@ -142,6 +247,20 @@
return km.toJSON();
};
+ /*
+ POST /collect/:result_id
+ POST /peek
+ POST /?text_id=...
+ POST /:text_id/
+
+ PUT /:text_id
+
+ DELETE /:text_id
+ DELETE /:corpus_sigle
+ DELETE /:corpus_sigle/:doc_sigle
+ DELETE /:corpus_sigle/:doc_sigle/:text_sigle
+ */
+
@POST
@Path("/collection")
@Produces(MediaType.APPLICATION_JSON)
@@ -156,4 +275,27 @@
return "{}";
};
+
+
+
+ // Interceptor class
+ public class GZIPReaderInterceptor implements ReaderInterceptor {
+ @Override
+ public Object aroundReadFrom(ReaderInterceptorContext context)
+ throws IOException, WebApplicationException {
+ final InputStream originalInputStream = context.getInputStream();
+ context.setInputStream(new GZIPInputStream(originalInputStream));
+ return context.proceed();
+ };
+ };
+
+ public class GZIPWriterInterceptor implements WriterInterceptor {
+ @Override
+ public void aroundWriteTo(WriterInterceptorContext context)
+ throws IOException, WebApplicationException {
+ final OutputStream outputStream = context.getOutputStream();
+ context.setOutputStream(new GZIPOutputStream(outputStream));
+ context.proceed();
+ };
+ };
};
diff --git a/src/main/java/de/ids_mannheim/korap/util/KorapString.java b/src/main/java/de/ids_mannheim/korap/util/KorapString.java
new file mode 100644
index 0000000..1a653de
--- /dev/null
+++ b/src/main/java/de/ids_mannheim/korap/util/KorapString.java
@@ -0,0 +1,24 @@
+package de.ids_mannheim.korap.util;
+
+import java.io.*;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.nio.charset.Charset;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * @author Nils Diewald
+ *
+ * A collection of methods to deal with Strings.
+ */
+public class KorapString {
+
+ public static String StringfromFile(String path, Charset encoding) throws IOException {
+ byte[] encoded = Files.readAllBytes(Paths.get(path));
+ return new String(encoded, encoding);
+ };
+
+ public static String StringfromFile (String path) throws IOException {
+ return StringfromFile(path, StandardCharsets.UTF_8);
+ };
+};
diff --git a/src/test/java/de/ids_mannheim/korap/server/PingTest.java b/src/test/java/de/ids_mannheim/korap/server/PingTest.java
deleted file mode 100644
index 12fad2c..0000000
--- a/src/test/java/de/ids_mannheim/korap/server/PingTest.java
+++ /dev/null
@@ -1,50 +0,0 @@
-package de.ids_mannheim.korap.server;
-
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.ClientBuilder;
-import javax.ws.rs.client.WebTarget;
-
-import org.glassfish.grizzly.http.server.HttpServer;
-
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Test;
-import static org.junit.Assert.assertEquals;
-
-import de.ids_mannheim.korap.KorapNode;
-
-public class PingTest {
-
- private HttpServer server;
- private WebTarget target;
-
- @Before
- public void setUp() throws Exception {
- // start the server
- server = KorapNode.startServer();
- // create the client
- Client c = ClientBuilder.newClient();
-
- // uncomment the following line if you want to enable
- // support for JSON in the client (you also have to uncomment
- // dependency on jersey-media-json module in pom.xml and Main.startServer())
- // --
- // c.configuration().enable(new org.glassfish.jersey.media.json.JsonJaxbFeature());
-
- target = c.target(KorapNode.BASE_URI);
- }
-
- @After
- public void tearDown() throws Exception {
- server.stop();
- }
-
- /**
- * Test to see that the message "Gimme 5 minutes, please!" is sent in the response.
- */
- @Test
- public void testPing() {
- String responseMsg = target.path("ping").request().get(String.class);
- assertEquals("Gimme 5 minutes, please!", responseMsg);
- }
-}
diff --git a/src/test/java/de/ids_mannheim/korap/server/ResourceTest.java b/src/test/java/de/ids_mannheim/korap/server/ResourceTest.java
new file mode 100644
index 0000000..5f2365f
--- /dev/null
+++ b/src/test/java/de/ids_mannheim/korap/server/ResourceTest.java
@@ -0,0 +1,93 @@
+package de.ids_mannheim.korap.server;
+
+/*
+ http://harryjoy.com/2012/09/08/simple-rest-client-in-java/
+*/
+import java.io.*;
+
+import javax.ws.rs.client.Client;
+import javax.ws.rs.client.ClientBuilder;
+import javax.ws.rs.client.WebTarget;
+import javax.ws.rs.client.Entity;
+
+import org.glassfish.grizzly.http.server.HttpServer;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import static org.junit.Assert.assertEquals;
+
+import java.io.FileInputStream;
+
+import de.ids_mannheim.korap.KorapNode;
+import de.ids_mannheim.korap.server.KorapResponse;
+import static de.ids_mannheim.korap.util.KorapString.*;
+
+public class ResourceTest {
+
+ private HttpServer server;
+ private WebTarget target;
+
+ @Before
+ public void setUp() throws Exception {
+ // start the server
+ server = KorapNode.startServer("milena", (String) null);
+ // create the client
+ Client c = ClientBuilder.newClient();
+
+ // uncomment the following line if you want to enable
+ // support for JSON in the client (you also have to uncomment
+ // dependency on jersey-media-json module in pom.xml and Main.startServer())
+ // --
+ // c.configuration().enable(new org.glassfish.jersey.media.json.JsonJaxbFeature());
+
+ target = c.target(KorapNode.BASE_URI);
+ };
+
+ @After
+ public void tearDown() throws Exception {
+ server.stop();
+ };
+
+ /**
+ * Test to see that the message "Gimme 5 minutes, please!" is sent in the response.
+ */
+ @Test
+ public void testPing() {
+ String responseMsg = target.path("ping").request().get(String.class);
+ assertEquals("Gimme 5 minutes, please!", responseMsg);
+ };
+
+ @Test
+ public void testResource() throws IOException {
+ int unstaged = 1;
+ for (String i : new String[] {"00001",
+ "00002",
+ "00003",
+ "00004",
+ "00005",
+ "00006",
+ "02439"
+ }) {
+
+ String json = StringfromFile(getClass().getResource("/wiki/" + i + ".json").getFile());
+ KorapResponse kresp = target.path("/" + i).
+ request("application/json").
+ put(Entity.json(json), KorapResponse.class);
+
+ assertEquals(kresp.getNode(), "milena");
+ /*
+ assertNull(kresp.getErr());
+ assertNull(kresp.getErrstr());
+ */
+ assertEquals(kresp.getUnstaged(), unstaged++);
+ assertEquals(kresp.getVersion(), "0.42");
+ };
+
+ KorapResponse kresp = target.path("/").
+ request("application/json").
+ post(Entity.text(""), KorapResponse.class);
+ assertEquals(kresp.getNode(), "milena");
+ assertEquals(kresp.getMsg(), "Unstaged data was committed");
+ };
+};