Added search timeout in meta query.
Change-Id: I7811d4ebc9b1f61f55a13e1b8a44e49b7fb0ea00
diff --git a/core/Changes b/core/Changes
index 91f2034..210c625 100644
--- a/core/Changes
+++ b/core/Changes
@@ -1,3 +1,7 @@
+# version 0.61.5
+17/11/2018
+ - Added search timeout in meta query (margaretha)
+
version 0.61.4
14/11/2018
- Integrated lite and full services and controllers in core (margaretha)
diff --git a/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java b/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
index 6e7c17e..2e43a33 100644
--- a/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
+++ b/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
@@ -31,6 +31,8 @@
@Service
public class SearchService {
+ private static final boolean DEBUG = false;
+
private static Logger jlog = LogManager.getLogger(SearchService.class);
@Autowired
@@ -106,13 +108,14 @@
KustvaktConfiguration.BACKENDS eng = this.config.chooseBackend(engine);
User user = createUser(username, headers);
-
+ CorpusAccess corpusAccess = user.getCorpusAccess();
+
QuerySerializer serializer = new QuerySerializer();
serializer.setQuery(q, ql, v);
if (cq != null) serializer.setCollection(cq);
MetaQueryBuilder meta = createMetaQuery(pageIndex, pageInteger, ctx,
- pageLength, cutoff);
+ pageLength, cutoff, corpusAccess);
if (fields != null && !fields.isEmpty())
meta.addEntry("fields", fields);
serializer.setMeta(meta.raw());
@@ -125,7 +128,9 @@
String query =
this.rewriteHandler.processQuery(serializer.toJSON(), user);
- jlog.info("the serialized query " + query);
+ if (DEBUG){
+ jlog.debug("the serialized query " + query);
+ }
String result;
if (eng.equals(KustvaktConfiguration.BACKENDS.NEO4J)) {
@@ -141,7 +146,7 @@
private MetaQueryBuilder createMetaQuery (Integer pageIndex,
Integer pageInteger, String ctx, Integer pageLength,
- Boolean cutoff) {
+ Boolean cutoff, CorpusAccess corpusAccess) {
MetaQueryBuilder meta = new MetaQueryBuilder();
meta.addEntry("startIndex", pageIndex);
meta.addEntry("startPage", pageInteger);
@@ -153,6 +158,13 @@
// cutoff);
// fixme: should only apply to CQL queries per default!
// meta.addEntry("itemsPerResource", 1);
+
+ if (corpusAccess.equals(CorpusAccess.FREE)){
+ meta.addEntry("timeout", 10000);
+ }
+ else{
+ meta.addEntry("timeout", 90000);
+ }
return meta;
}
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/SearchKrill.java b/core/src/main/java/de/ids_mannheim/korap/web/SearchKrill.java
index f52bbdb..f3505ce 100644
--- a/core/src/main/java/de/ids_mannheim/korap/web/SearchKrill.java
+++ b/core/src/main/java/de/ids_mannheim/korap/web/SearchKrill.java
@@ -32,6 +32,8 @@
private final static Logger jlog = LogManager
.getLogger(SearchKrill.class);
+ private static final boolean DEBUG = false;
+
// Temporary - shouldn't be here.
String indexDir = "/data/prep_corpus/index/";
String i = "/Users/hanl/Projects/prep_corpus";
@@ -46,7 +48,7 @@
try {
if (path.equals(":temp:")) {
- this.index = new KrillIndex();
+ index = new KrillIndex();
}
else {
File f = new File(path);
@@ -55,7 +57,7 @@
jlog.error("Index not found: " + path + "!");
System.exit(-1);
}
- this.index = new KrillIndex(new MMapDirectory(Paths.get(path)));
+ index = new KrillIndex(new MMapDirectory(Paths.get(path)));
};
}
catch (IOException e) {
@@ -64,7 +66,7 @@
};
public KrillIndex getIndex () {
- return this.index;
+ return index;
};
@@ -76,9 +78,16 @@
* filters.
*/
public String search (String json) {
- jlog.trace(json);
- if (this.index != null)
- return new Krill(json).apply(this.index).toJsonString();
+ if (DEBUG){
+ jlog.debug(json);
+ }
+ if (index != null){
+ String result = new Krill(json).apply(index).toJsonString();
+// if (DEBUG){
+ jlog.debug(result);
+// }
+ return result;
+ }
Result kr = new Result();
kr.addError(601, "Unable to find index");
return kr.toJsonString();
@@ -94,9 +103,11 @@
*/
@Deprecated
public String searchTokenList (String json) {
- jlog.trace(json);
- if (this.index != null)
- return new Krill(json).apply(this.index).toTokenListJsonString();
+ if (DEBUG){
+ jlog.debug(json);
+ }
+ if (index != null)
+ return new Krill(json).apply(index).toTokenListJsonString();
Result kr = new Result();
kr.addError(601, "Unable to find index");
return kr.toJsonString();
@@ -114,9 +125,9 @@
*/
public String getMatch (String id, Pattern licensePattern) {
Match km;
- if (this.index != null) {
+ if (index != null) {
try {
- km = this.index.getMatch(id);
+ km = index.getMatch(id);
String availability = km.getAvailability();
if (licensePattern!=null && availability != null){
Matcher m = licensePattern.matcher(availability);
@@ -152,7 +163,7 @@
MetaFields meta;
// No index found
- if (this.index == null) {
+ if (index == null) {
meta = new MetaFields(id);
meta.addError(601, "Unable to find index");
}
@@ -161,7 +172,7 @@
else {
//Get fields
- meta = this.index.getFields(id);
+ meta = index.getFields(id);
};
return meta.toJsonString();
};
@@ -173,9 +184,9 @@
boolean includeHighlights, boolean sentenceExpansion,
Pattern licensePattern) {
Match km;
- if (this.index != null) {
+ if (index != null) {
try {
- km = this.index.getMatchInfo(id, "tokens", true, foundries,
+ km = index.getMatchInfo(id, "tokens", true, foundries,
layers, includeSpans, includeHighlights,
sentenceExpansion);
String availability = km.getAvailability();
@@ -227,7 +238,7 @@
boolean includeSpans, boolean includeHighlights,
boolean sentenceExpansion) {
- if (this.index != null) {
+ if (index != null) {
try {
/*
For multiple foundries/layers use
@@ -240,7 +251,7 @@
boolean includeHighlights,
boolean extendToSentence
*/
- return this.index.getMatchInfo(id, "tokens", foundry, layer,
+ return index.getMatchInfo(id, "tokens", foundry, layer,
includeSpans, includeHighlights, sentenceExpansion)
.toJsonString();
}
@@ -265,14 +276,16 @@
* JSON-LD string with potential meta filters.
*/
public String getStatistics (String json) {
- if (this.index == null) {
+ if (index == null) {
return "{\"documents\" : -1, error\" : \"No index given\" }";
};
// Define a virtual corpus
KrillCollection kc;
if (json != null && !json.equals("")) {
- jlog.trace(json);
+ if (DEBUG){
+ jlog.debug(json);
+ }
// Create Virtual collection from json search
kc = new KrillCollection(json);
@@ -286,7 +299,7 @@
};
// Set index
- kc.setIndex(this.index);
+ kc.setIndex(index);
long docs = 0, tokens = 0, sentences = 0, paragraphs = 0;
// Get numbers from index (currently slow)
try {
diff --git a/full/Changes b/full/Changes
index d2d6755..c9be960 100644
--- a/full/Changes
+++ b/full/Changes
@@ -1,3 +1,7 @@
+# version 0.61.5
+17/11/2018
+ - Added a search timeout test (margaretha)
+
# version 0.61.4
14/11/2018
- Integrated lite and full services and controllers in core (margaretha)
diff --git a/full/pom.xml b/full/pom.xml
index 71252ac..9d0a1d9 100644
--- a/full/pom.xml
+++ b/full/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-full</artifactId>
- <version>0.61.4</version>
+ <version>0.61.5</version>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/full/src/test/java/de/ids_mannheim/korap/web/SearchKrillTest.java b/full/src/test/java/de/ids_mannheim/korap/web/SearchKrillTest.java
index 34dd3be..ec5561b 100644
--- a/full/src/test/java/de/ids_mannheim/korap/web/SearchKrillTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/web/SearchKrillTest.java
@@ -4,74 +4,64 @@
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import javax.annotation.PostConstruct;
+
import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.databind.JsonNode;
import de.ids_mannheim.korap.KrillIndex;
-import de.ids_mannheim.korap.config.BeanConfigTest;
import de.ids_mannheim.korap.config.KustvaktConfiguration;
-import de.ids_mannheim.korap.config.FullConfiguration;
+import de.ids_mannheim.korap.config.SpringJerseyTest;
import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.query.serialize.MetaQueryBuilder;
import de.ids_mannheim.korap.query.serialize.QuerySerializer;
import de.ids_mannheim.korap.utils.JsonUtils;
-import de.ids_mannheim.korap.web.SearchKrill;
/**
* Created by hanl on 02.06.16.
+ *
+ * Updated by margaretha
*/
-public class SearchKrillTest extends BeanConfigTest {
+public class SearchKrillTest extends SpringJerseyTest {
- @Override
- public void initMethod () throws KustvaktException {
-
- }
-
-
- @Test
- public void testInit () {
- KustvaktConfiguration config = helper().getContext().getConfiguration();
- SearchKrill krill = new SearchKrill(config.getIndexDir());
+ @Autowired
+ KustvaktConfiguration config;
+
+ SearchKrill krill = null;
+
+ @PostConstruct
+ private void createKrill () {
+ krill = new SearchKrill(config.getIndexDir());
assertNotNull(krill);
}
-
@Test
- public void testIndex () {
- KustvaktConfiguration config = helper().getContext().getConfiguration();
- SearchKrill krill = new SearchKrill(config.getIndexDir());
- assertNotNull(krill);
-
+ public void testIndex () throws KustvaktException {
KrillIndex index = krill.getIndex();
assertNotNull(index);
}
-
@Test
public void testDocSize () {
- KustvaktConfiguration config = helper().getContext().getConfiguration();
- SearchKrill krill = new SearchKrill(config.getIndexDir());
- assertNotNull(krill);
assertNotEquals(0, krill.getIndex().numberOf("documents"));
}
- @Test
+ @Test
public void testMatchInfo () throws KustvaktException {
- FullConfiguration config = helper().getContext().getConfiguration();
- SearchKrill krill = new SearchKrill(config.getIndexDir());
- assertNotNull(krill);
- String matchinfo = krill.getMatch("WPD/AAA.00002/p169-197", config.getFreeLicensePattern());
- JsonNode node = JsonUtils.readTree(matchinfo);
- assertEquals("Invalid match identifier", node.at("/errors/0/1").asText());
- }
+ String matchinfo = krill.getMatch("WPD/AAA.00002/p169-197",
+ config.getFreeLicensePattern());
+ JsonNode node = JsonUtils.readTree(matchinfo);
+ assertEquals("Invalid match identifier",
+ node.at("/errors/0/1").asText());
+ }
@Test
public void testSearch () throws KustvaktException {
QuerySerializer s = new QuerySerializer();
s.setQuery("[orth=der]", "poliqarp");
- KustvaktConfiguration config = helper().getContext().getConfiguration();
- SearchKrill krill = new SearchKrill(config.getIndexDir());
String result = krill.search(s.toJSON());
JsonNode node = JsonUtils.readTree(result);
@@ -79,4 +69,22 @@
assertNotEquals(0, node.at("/matches").size());
}
+ @Test
+ public void testTimeOut () throws KustvaktException {
+ QuerySerializer s = new QuerySerializer();
+ s.setQuery("[orth=der]", "poliqarp");
+// s.setQuery("node ->malt/d[func=/.*/] node", "annis");
+
+ MetaQueryBuilder meta = new MetaQueryBuilder();
+ meta.addEntry("timeout", 1);
+ s.setMeta(meta);
+
+ String query = s.toJSON();
+ JsonNode node = JsonUtils.readTree(query);
+ assertEquals(1, node.at("/meta/timeout").asInt());
+
+ String result = krill.search(query);
+ node = JsonUtils.readTree(result);
+ assertEquals(true,node.at("/meta/timeExceeded").asBoolean());
+ }
}
diff --git a/full/src/test/java/de/ids_mannheim/korap/web/controller/AnnotationControllerTest.java b/full/src/test/java/de/ids_mannheim/korap/web/controller/AnnotationControllerTest.java
index f976e32..33f34f0 100644
--- a/full/src/test/java/de/ids_mannheim/korap/web/controller/AnnotationControllerTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/web/controller/AnnotationControllerTest.java
@@ -1,7 +1,6 @@
package de.ids_mannheim.korap.web.controller;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import java.util.Iterator;