Merge "Ignore bin"
diff --git a/Changes b/Changes
index b946585..a6f30b2 100644
--- a/Changes
+++ b/Changes
@@ -1,4 +1,4 @@
-0.51 2015-03-04
+0.51 2015-03-17
- This is a major version (prepared for the GitHub release)
- [cleanup] Changed groupID to "de.ids_mannheim.korap",
renamed korap-style.xml to Format.xml (diewald)
@@ -9,6 +9,10 @@
- [bugfix] Updated SpanRelationQuery (margaretha)
- [cleanup] Autoformat (diewald)
- [documentation] References added to the Readme (diewald)
+ - [bugfix] Improved failing for missing property file (kupietz)
+ - [bugfix] Fixed tests for server responses to not use
+ Jackson deserialization (diewald)
+ - [cleanup] No more jersey logging in tests (diewald)
0.50.1 2015-03-02
- [feature] Deserialization of arbitrary elements with
diff --git a/Errorcodes b/Errorcodes
index 537da15..524924a 100644
--- a/Errorcodes
+++ b/Errorcodes
@@ -1,4 +1,4 @@
-* 600 - 699 - Lucene Backend error codes
+* 600 - 699 - Krill server error codes
600: "Unable to read index"
601: "Unable to find index"
602: "Unable to add document to index"
@@ -10,8 +10,9 @@
680: "Server is up and running!"
681: "Document was added successfully", document id
682: "Response time exceeded"
+683: "Staged data committed"
-* 700 - 799 - Coral Deserialization errors
+* 700 - 799 - KoralQuery Deserialization errors
700: "No Query given"
701: "JSON-LD group has no @type attribute"
702: "Boundary definition is invalid"
diff --git a/Readme.md b/Readme.md
index 0a9feba..0dac45b 100644
--- a/Readme.md
+++ b/Readme.md
@@ -67,13 +67,6 @@
$ mvn clean test
```
-
-To start the server ...
-
-```
-$ mvn compile exec:java
-```
-
## Caveats
Krill operates on tokens and is limited to a single tokenization stream.
diff --git a/pom.xml b/pom.xml
index 83dcd54..821b2f1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -84,6 +84,11 @@
<artifactId>slf4j-log4j12</artifactId>
<version>1.7.5</version>
</dependency>
+ <dependency>
+ <groupId>org.slf4j</groupId>
+ <artifactId>jul-to-slf4j</artifactId>
+ <version>1.7.5</version>
+ </dependency>
<!-- SQLite for database connection tests -->
<dependency>
@@ -149,6 +154,11 @@
</dependency>
<!-- JSON support in Jersey -->
+ <dependency>
+ <groupId>com.fasterxml.jackson.jaxrs</groupId>
+ <artifactId>jackson-jaxrs-json-provider</artifactId>
+ <version>2.4.4</version>
+ </dependency>
<!--
<dependency>
<groupId>org.glassfish.jersey.media</groupId>
@@ -165,11 +175,6 @@
<version>2.16</version>
</dependency>
-->
- <dependency>
- <groupId>com.fasterxml.jackson.jaxrs</groupId>
- <artifactId>jackson-jaxrs-json-provider</artifactId>
- <version>2.4.4</version>
- </dependency>
<!-- JSON support using Jackson -->
<!-- see https://github.com/FasterXML/jackson-core -->
diff --git a/src/main/java/de/ids_mannheim/korap/KrillIndex.java b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
index a425f36..cb604be 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
@@ -116,8 +116,8 @@
// Last line of defense against DOS
private int maxTermRelations = 100;
private int autoCommit = 500;
- private String version;
- private String name;
+ private String version = "unknown";
+ private String name = "Krill";
// Temp:
private IndexReader reader;
diff --git a/src/main/java/de/ids_mannheim/korap/server/Node.java b/src/main/java/de/ids_mannheim/korap/server/Node.java
index 8b4de0f..b4d0068 100644
--- a/src/main/java/de/ids_mannheim/korap/server/Node.java
+++ b/src/main/java/de/ids_mannheim/korap/server/Node.java
@@ -10,6 +10,8 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.util.logging.LogManager;
+import org.slf4j.bridge.SLF4JBridgeHandler;
import java.net.URI;
import java.beans.PropertyVetoException;
@@ -20,31 +22,37 @@
import com.mchange.v2.c3p0.*;
/**
- * Standalone REST-Service for the Lucene Search Backend.
+ * Standalone REST-Service for the Krill node.
+ * Reads a property file at <tt>krill.properties</tt>.
+ * Defaults to port <tt>9876</tt> if no information is given,
+ * and an unprotected in-memory SQLite database for collections.
*
* @author diewald
*/
public class Node {
// Base URI the Grizzly HTTP server will listen on
- public static String BASE_URI = "http://localhost:8080/";
+ public static String BASE_URI = "http://localhost:9876/";
+ private static String propFile = "krill.properties";
+
// Logger
private final static Logger log = LoggerFactory.getLogger(Node.class);
// Index
private static KrillIndex index;
+
+ // Database
private static ComboPooledDataSource cpds;
- private static String path, name = "unknown";
-
+ private static String path = null;
+ private static String name = "unknown";
private static String dbUser, dbPwd;
-
- private static String dbClass = "org.sqlite.JDBC", dbURL = "jdbc:sqlite:";
+ private static String dbClass = "org.sqlite.JDBC";
+ private static String dbURL = "jdbc:sqlite:";
/*
- * Todo: Add shutdown hook,
- * Then also close cdps.close();
+ * Todo: Close cdps.close() on shutdown.
* see: https://10.0.10.12/trac/korap/browser/KorAP-modules/KorAP-REST/src/main/java/de/ids_mannheim/korap/web/Application.java
* https://10.0.10.12/trac/korap/browser/KorAP-modules/KorAP-REST/src/main/java/de/ids_mannheim/korap/web/ShutdownHook.java
*/
@@ -52,52 +60,38 @@
/**
* Starts Grizzly HTTP server exposing JAX-RS
* resources defined in this application.
+ * This will load a <tt>krill.properties</tt> property file.
*
* @return Grizzly HTTP server.
*/
public static HttpServer startServer () {
-
- // Load configuration
- URL resUrl = Node.class.getClassLoader().getResource("krill.properties");
- if (resUrl == null) {
- log.error("Cannot find \"krill.properties\". Please create it "
- +"using \"krill.properties.info\" as template. Terminating.");
- System.exit(1);
- }
- try {
- InputStream file = new FileInputStream(resUrl.getFile());
- Properties prop = new Properties();
- prop.load(file);
-
- // Node properties
- path = prop.getProperty("krill.indexDir", path);
- name = prop.getProperty("krill.server.name", name);
- BASE_URI = prop.getProperty("krill.server.baseURI", BASE_URI);
-
- // Database properties
- dbUser = prop.getProperty("krill.db.user", dbUser);
- dbPwd = prop.getProperty("krill.db.pwd", dbPwd);
- dbClass = prop.getProperty("krill.db.class", dbClass);
- dbURL = prop.getProperty("krill.db.jdbcURL", dbURL);
-
- }
- catch (IOException e) {
- log.error(e.getLocalizedMessage());
- };
+ _loadResourceProperties();
// create a resource config that scans for JAX-RS resources and providers
// in de.ids_mannheim.korap.server package
final ResourceConfig rc = new ResourceConfig()
- .packages("de.ids_mannheim.korap.server");
+ .packages("de.ids_mannheim.korap.server");
// create and start a new instance of grizzly http server
// exposing the Jersey application at BASE_URI
- return GrizzlyHttpServerFactory.createHttpServer(URI.create(BASE_URI),
- rc);
+ return GrizzlyHttpServerFactory.createHttpServer(
+ URI.create(BASE_URI), rc
+ );
};
-
+ /**
+ * Starts Grizzly HTTP server exposing JAX-RS
+ * resources defined in this application.
+ * Mainly used for testing.
+ *
+ * @param nodeName The name of the node.
+ * @param indexPath The path of the Lucene index.
+ *
+ * @return Grizzly {@link HttpServer} server.
+ */
public static HttpServer startServer (String nodeName, String indexPath) {
+ LogManager.getLogManager().reset();
+ SLF4JBridgeHandler.install();
// create a resource config that scans for JAX-RS resources and providers
// in de.ids_mannheim.korap.server package
@@ -115,25 +109,32 @@
/**
- * Main method.
+ * Runner method for Krill node.
*
- * @param args
+ * @param args No special arguments required.
* @throws IOException
*/
public static void main (String[] args) throws IOException {
- // WADL available at BASE_URI + application.wadl
+ // WADL available at BASE_URI + application.wadl
+ // Start the server with krill properties or given defaults
final HttpServer server = startServer();
// Establish shutdown hook
- Runtime.getRuntime().addShutdownHook(new Thread(new Runnable() {
- @Override
- public void run () {
- log.info("Stop Server");
- // staaahp!
- server.stop();
- }
- }, "shutdownHook"));
+ Runtime.getRuntime().addShutdownHook(
+ new Thread(
+ new Runnable() {
+
+ @Override
+ public void run () {
+ log.info("Stop Server");
+ server.stop();
+ if (cpds != null)
+ cpds.close();
+ };
+
+ }, "shutdownHook")
+ );
// Start server
try {
@@ -147,25 +148,49 @@
};
- // What's the servers name?
+ /**
+ * Get the name of the node.
+ * The name is unique in the cluster and should be persistent.
+ *
+ * @return The unique name of the node.
+ */
public static String getName () {
return name;
};
- // What is the server listening on?
+ /**
+ * Get the URI (incl. port) the node is listening on.
+ *
+ * @return The URI the node is listening on.
+ */
public static String getListener () {
return BASE_URI;
};
- // Get database pool
+ /**
+ * Shut down the database pool.
+ */
+ public static void closeDBPool () {
+ if (cpds != null)
+ cpds.close();
+ };
+
+
+ /**
+ * Get the associated database pool
+ * for match collection.
+ *
+ * @return The CPDS {@link ComboPooledDataSource} object.
+ */
public static ComboPooledDataSource getDBPool () {
// Pool already initiated
if (cpds != null)
return cpds;
+ // Initiate pool
try {
// Parameters are defined in the property file
@@ -186,7 +211,11 @@
};
- // Get Lucene Index
+ /**
+ * Get the associuated {@link KrillIndex}.
+ *
+ * @return The associated {@link KrillIndex}.
+ */
public static KrillIndex getIndex () {
// Index already instantiated
@@ -196,10 +225,13 @@
try {
// Get a temporary index
- if (path == null)
+ if (path == null) {
+
// Temporary index
index = new KrillIndex();
+ }
+ // Get a MMap directory index
else {
File file = new File(path);
@@ -219,4 +251,45 @@
};
return null;
};
+
+
+ // Load properties from file
+ private static Properties _loadProperties (String propFile) {
+ try {
+ InputStream file = new FileInputStream(propFile);
+ Properties prop = new Properties();
+ prop.load(file);
+
+ // Node properties
+ path = prop.getProperty("krill.indexDir", path);
+ name = prop.getProperty("krill.server.name", name);
+ BASE_URI = prop.getProperty("krill.server.baseURI", BASE_URI);
+
+ // Database properties
+ dbUser = prop.getProperty("krill.db.user", dbUser);
+ dbPwd = prop.getProperty("krill.db.pwd", dbPwd);
+ dbClass = prop.getProperty("krill.db.class", dbClass);
+ dbURL = prop.getProperty("krill.db.jdbcURL", dbURL);
+ return prop;
+ }
+ catch (IOException e) {
+ log.error(e.getLocalizedMessage());
+ };
+ return null;
+ };
+
+
+ // Load properties from resource file
+ private static Properties _loadResourceProperties () {
+
+ // Load configuration
+ URL resUrl = Node.class.getClassLoader().getResource(propFile);
+ if (resUrl == null) {
+ log.error("Cannot find {}. Please create it using \"{}.info\" as template.",
+ propFile, propFile);
+ return null;
+ };
+
+ return _loadProperties(resUrl.getFile());
+ };
};
diff --git a/src/main/java/de/ids_mannheim/korap/server/Resource.java b/src/main/java/de/ids_mannheim/korap/server/Resource.java
index 4b4fa31..be13ee7 100644
--- a/src/main/java/de/ids_mannheim/korap/server/Resource.java
+++ b/src/main/java/de/ids_mannheim/korap/server/Resource.java
@@ -20,16 +20,16 @@
import javax.ws.rs.WebApplicationException;
import de.ids_mannheim.korap.server.Node;
-import de.ids_mannheim.korap.KrillIndex;
import de.ids_mannheim.korap.Krill;
+import de.ids_mannheim.korap.KrillIndex;
import de.ids_mannheim.korap.KrillCollection;
import de.ids_mannheim.korap.response.Result;
import de.ids_mannheim.korap.response.Match;
import de.ids_mannheim.korap.response.Response;
-import de.ids_mannheim.korap.index.FieldDocument;
-import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.response.MatchCollector;
import de.ids_mannheim.korap.response.collector.MatchCollectorDB;
+import de.ids_mannheim.korap.util.QueryException;
+import de.ids_mannheim.korap.index.FieldDocument;
import java.util.List;
import java.util.regex.Pattern;
@@ -45,16 +45,14 @@
/**
- * Root resource (exposed at root path)
+ * Root resource (exposed at root path) of the Krill node.
* The responses only represent JSON responses, although HTML
- * responses
- * may be handy.
+ * responses may be handy.
*
- * @author Nils Diewald
- *
- * Look at
- * http://www.mkyong.com/webservices/jax-rs/json-example
- * -with-jersey-jackson/
+ * @author diewald
+ */
+/* Look at
+ * http://www.mkyong.com/webservices/jax-rs/json-example-with-jersey-jackson/
*/
@Path("/")
public class Resource {
@@ -69,20 +67,8 @@
// Slightly based on String::BooleanSimple
static Pattern p = Pattern
- .compile("\\s*(?i:false|no|inactive|disabled|off|n|neg(?:ative)?|not|null|undef)\\s*");
-
-
- // Check if a string is meant to represent null
- private static boolean isNull (String value) {
- if (value == null)
- return true;
-
- Matcher m = p.matcher(value);
- if (m.matches())
- return true;
-
- return false;
- };
+ .compile("\\s*(?i:false|no|inactive|disabled|" +
+ "off|n|neg(?:ative)?|not|null|undef)\\s*");
/**
@@ -91,19 +77,19 @@
@GET
@Produces(MediaType.APPLICATION_JSON)
public String info () {
- KrillIndex index = Node.getIndex();
Response kresp = new Response();
kresp.setNode(Node.getName());
+ kresp.setListener(Node.getListener());
+
+ // Get index
+ KrillIndex index = Node.getIndex();
kresp.setName(index.getName());
kresp.setVersion(index.getVersion());
-
- kresp.setListener(Node.getListener());
- long texts = -1;
/*
- kresp.addMessage(
+ kresp.addMessage(
"Number of documents in the index",
String.parseLong(index.numberOf("documents"))
- );
+ );
*/
kresp.addMessage(680, "Server is up and running!");
return kresp.toJsonString();
@@ -134,7 +120,6 @@
*/
// Todo: Parameter for server node
-
if (DEBUG)
log.trace("Added new document with unique identifier {}", uid);
@@ -197,6 +182,7 @@
// There are documents to commit
try {
index.commit();
+ kresp.addMessage(683, "Staged data committed");
}
catch (IOException e) {
// Set HTTP to ???
@@ -373,22 +359,22 @@
boolean includeSpans = false, includeHighlights = true, extendToSentence = false, info = false;
// Optional query parameter "info" for more information on the match
- if (!isNull(qp.getFirst("info")))
+ if (!_isNull(qp.getFirst("info")))
info = true;
// Optional query parameter "spans" for span information inclusion
- if (!isNull(qp.getFirst("spans"))) {
+ if (!_isNull(qp.getFirst("spans"))) {
includeSpans = true;
info = true;
};
// Optional query parameter "highlights" for highlight information inclusion
String highlights = qp.getFirst("highlights");
- if (highlights != null && isNull(highlights))
+ if (highlights != null && _isNull(highlights))
includeHighlights = false;
// Optional query parameter "extended" for sentence expansion
- if (!isNull(qp.getFirst("extended")))
+ if (!_isNull(qp.getFirst("extended")))
extendToSentence = true;
List<String> foundries = qp.get("foundry");
@@ -468,4 +454,17 @@
context.proceed();
};
};
+
+
+ // Check if a string is meant to represent null
+ private static boolean _isNull (String value) {
+ if (value == null)
+ return true;
+
+ Matcher m = p.matcher(value);
+ if (m.matches())
+ return true;
+
+ return false;
+ };
};
diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties
index 9bd7ad2..5fed33b 100644
--- a/src/main/resources/log4j.properties
+++ b/src/main/resources/log4j.properties
@@ -1,4 +1,4 @@
-# log4j.rootLogger = ERROR, stdout
+log4j.rootLogger = ERROR, stdout
# Queries:
# log4j.logger.de.ids_mannheim.korap.query.SpanNextQuery = TRACE, stdout
@@ -35,8 +35,11 @@
# Tests:
# log4j.logger.de.ids_mannheim.korap.index.TestSegmentIndex = TRACE, stdout
-log4j.appender.stdout=org.apache.log4j.ConsoleAppender
-log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+# Server
+# log4j.category.org.glassfish.jersey = TRACE, stdout
+
+log4j.appender.stdout = org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern = %5p (%F:%L) -> %m%n
# log4j.appender.stdout.Target=System.out
diff --git a/src/test/java/de/ids_mannheim/korap/server/TestResource.java b/src/test/java/de/ids_mannheim/korap/server/TestResource.java
index b69fb31..4593320 100644
--- a/src/test/java/de/ids_mannheim/korap/server/TestResource.java
+++ b/src/test/java/de/ids_mannheim/korap/server/TestResource.java
@@ -10,13 +10,11 @@
import javax.ws.rs.client.Entity;
import org.glassfish.grizzly.http.server.HttpServer;
-import com.fasterxml.jackson.jaxrs.annotation.JacksonFeatures;
import static org.junit.Assert.*;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import org.junit.Ignore;
import java.io.FileInputStream;
@@ -25,6 +23,9 @@
import de.ids_mannheim.korap.response.Response;
import static de.ids_mannheim.korap.util.KrillString.*;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
/**
* @author diewald
@@ -34,6 +35,7 @@
private HttpServer server;
private WebTarget target;
+ ObjectMapper mapper = new ObjectMapper();
@Before
public void setUp () throws Exception {
@@ -49,7 +51,7 @@
// c.configuration().enable(com.sun.jersey.api.json.POJOMappingFeature());
// c.configuration().enable(new org.glassfish.jersey.media.json.JsonJaxbFeature());
- // c.register(JacksonFeature.class);
+ // c.register(JacksonFeatures.class);
// c.register(com.fasterxml.jackson.jaxrs.annotation.JacksonFeatures.class);
/*
@@ -64,6 +66,7 @@
@After
public void tearDown () throws Exception {
server.stop();
+ Node.closeDBPool();
};
@@ -77,10 +80,19 @@
assertEquals("Gimme 5 minutes, please!", responseMsg);
};
+ @Test
+ public void testInfo () throws IOException {
+ String responseMsg = target.path("/").request().get(String.class);
+ JsonNode res = mapper.readTree(responseMsg);
+ assertEquals("milena", res.at("/node").asText());
+ assertEquals(680, res.at("/messages/0/0").asInt());
+ };
- @Ignore
+
+ @Test
public void testResource () throws IOException {
- Response kresp;
+ String resp;
+ JsonNode res;
for (String i : new String[] { "00001", "00002", "00003", "00004",
"00005", "00006", "02439" }) {
@@ -91,13 +103,13 @@
Entity jsonE = Entity.json(json);
try {
- kresp = target.path("/index/" + i).request("application/json")
- .put(jsonE, Response.class);
+ // Put new documents to the index
+ resp = target.path("/index/" + i).request("application/json")
+ .put(jsonE, String.class);
- assertEquals(kresp.getNode(), "milena");
- assertFalse(kresp.hasErrors());
- assertFalse(kresp.hasWarnings());
- assertFalse(kresp.hasMessages());
+ res = mapper.readTree(resp);
+ assertEquals("milena", res.at("/node").asText());
+ assertEquals(681, res.at("/messages/0/0").asInt());
}
catch (Exception e) {
fail("Server response failed " + e.getMessage()
@@ -105,30 +117,27 @@
}
};
- kresp = target.path("/index").request("application/json")
- .post(Entity.text(""), Response.class);
- assertEquals(kresp.getNode(), "milena");
- assertFalse(kresp.hasErrors());
- assertFalse(kresp.hasWarnings());
- assertFalse(kresp.hasMessages());
+ resp = target.path("/index").request("application/json")
+ .post(Entity.text(""), String.class);
+ res = mapper.readTree(resp);
+ assertEquals("milena", res.at("/node").asText());
+ assertEquals(683, res.at("/messages/0/0").asInt());
};
- @Ignore
+ @Test
public void testCollection () throws IOException {
String json = getString(getClass().getResource(
- "/queries/bsp-uid-example.jsonld").getFile());
+ "/queries/bsp-uid-example.jsonld").getFile()
+ );
try {
- Response kresp = target.path("/").queryParam("uid", "1")
- .queryParam("uid", "4").request("application/json")
- .post(Entity.json(json), Response.class);
-
- assertEquals(2, kresp.getTotalResults());
- assertFalse(kresp.hasErrors());
- assertFalse(kresp.hasWarnings());
- assertFalse(kresp.hasMessages());
+ String resp = target.path("/").queryParam("uid", "1")
+ .queryParam("uid", "4").request("application/json")
+ .post(Entity.json(json), String.class);
+ JsonNode res = mapper.readTree(resp);
+ assertEquals(2, res.at("/totalResults").asInt());
}
catch (Exception e) {
fail("Server response failed: " + e.getMessage() + " (Known issue)");