Added clean timeout
diff --git a/CHANGES b/CHANGES
index 32c6468..372eb15 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,5 +1,6 @@
0.49 2014-11-08
- - [bugfix-todo] Rewrite bug when repeating expansions (margaretha)
+ - [bugfix] Rewrite works in repeating expansions (margaretha)
+ - [feature] Clean timeout with parameter support (diewald)
0.48 2014-11-07
- [feature] Retrieval of token lists (diewald)
diff --git a/pom.xml b/pom.xml
index de888a1..49175d1 100644
--- a/pom.xml
+++ b/pom.xml
@@ -7,7 +7,7 @@
install the indexer and the performancetests
$ mvn clean compile assembly:single
Then run e.g.
- $ java -jar target/KorAP-lucene-index-???-jar-with-dependencies.jar src/main/resources/korap.conf /data/hdd/lucene-new/WPD/
+ $ java -jar target/KorAP-lucene-index-X.XX.jar src/main/resources/korap.conf /data/hdd/lucene-new/WPD/
** Server
Start the server with
@@ -234,6 +234,7 @@
<excludes>
<exclude>**/benchmark/*</exclude>
<exclude>**/TestWPDIndex.java</exclude>
+ <exclude>**/TestRealIndex.java</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/src/main/java/de/ids_mannheim/korap/KorapIndex.java b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
index be9dd32..c145e38 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
@@ -347,18 +347,6 @@
this.closeReader();
};
- // Return the number of unstaged texts
- public boolean getUnstaged () throws IOException {
- if (commitCounter > 0)
- return true;
-
- // Open writer if not already opened
- if (this.writer == null)
- this.writer = new IndexWriter(this.directory, this.config);
-
- return this.writer.hasUncommittedChanges();
- };
-
// Get autoCommit valiue
public int autoCommit () {
return this.autoCommit;
@@ -1046,7 +1034,10 @@
return this.search(ks);
};
- // To be honest - the collection is already part of the KorapSearch!
+
+ /**
+ * Search the endpoint.
+ */
public KorapResult search (KorapSearch ks) {
if (DEBUG)
log.trace("Start search");
@@ -1101,6 +1092,14 @@
// Collect matches from atomic readers
ArrayList<KorapMatch> atomicMatches = new ArrayList<KorapMatch>(kr.itemsPerPage());
+ // Start time out thread
+ TimeOutThread tthread = new TimeOutThread();
+ tthread.start();
+ long timeout = ks.getTimeOut();
+
+ // See: http://www.ibm.com/developerworks/java/library/j-benchmark1/index.html
+ long t1 = System.nanoTime();
+
try {
// Rewrite query (for regex and wildcard queries)
@@ -1110,9 +1109,7 @@
query = (SpanQuery) rewrittenQuery;
};
- // See: http://www.ibm.com/developerworks/java/library/j-benchmark1/index.html
- long t1 = System.nanoTime();
-
+ // Todo: run this in a separated thread
for (AtomicReaderContext atomic : this.reader().leaves()) {
int oldLocalDocID = -1;
@@ -1130,8 +1127,7 @@
IndexReader lreader = atomic.reader();
- // TODO: Get document information from Cache!
-
+ // TODO: Get document information from Cache! Fieldcache?
for (; i < hits;i++) {
if (DEBUG)
@@ -1141,6 +1137,12 @@
if (!spans.next())
break;
+ // Timeout!
+ if (tthread.getTime() > timeout) {
+ kr.setTimeExceeded(true);
+ break;
+ };
+
int localDocID = spans.doc();
// Count hits per resource
@@ -1204,6 +1206,12 @@
if (limit > 0 && i >= limit)
break;
+ // Timeout!
+ if (tthread.getTime() > timeout) {
+ kr.setTimeExceeded(true);
+ break;
+ };
+
// Count hits per resource
if (itemsPerResource > 0) {
int localDocID = spans.doc();
@@ -1237,8 +1245,6 @@
atomicMatches.clear();
};
- kr.setBenchmark(t1, System.nanoTime());
-
if (itemsPerResource > 0)
kr.setItemsPerResource(itemsPerResource);
@@ -1249,6 +1255,12 @@
log.warn( e.getLocalizedMessage() );
};
+ // Stop timer thread
+ tthread.stopTimer();
+
+ // Calculate time
+ kr.setBenchmark(t1, System.nanoTime());
+
return kr;
};
@@ -1274,7 +1286,7 @@
// See: http://www.ibm.com/developerworks/java/library/j-benchmark1/index.html
long t1 = System.nanoTime();
- // Only load UID
+ // Only load UIDs
HashSet<String> fields = new HashSet<>(1);
fields.add("UID");
@@ -1293,6 +1305,7 @@
String uniqueDocIDString;;
int uniqueDocID = -1;
+ // start thread:
for (AtomicReaderContext atomic : this.reader().leaves()) {
int previousDocID = -1;
@@ -1348,6 +1361,7 @@
matchcount = 0;
};
};
+ // end thread
// Benchmark the collector
mc.setBenchmark(t1, System.nanoTime());
diff --git a/src/main/java/de/ids_mannheim/korap/KorapResult.java b/src/main/java/de/ids_mannheim/korap/KorapResult.java
index 5352028..0db50aa 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapResult.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapResult.java
@@ -47,6 +47,8 @@
private JsonNode request;
+ private boolean timeExceeded = false;
+
// Logger
// This is KorapMatch instead of KorapResult!
private final static Logger log = LoggerFactory.getLogger(KorapMatch.class);
@@ -188,6 +190,14 @@
return this.itemsPerResource;
};
+ public void setTimeExceeded (boolean timeout) {
+ this.timeExceeded = timeout;
+ };
+
+ public boolean getTimeExceeded () {
+ return this.timeExceeded;
+ };
+
public String getQuery () {
return this.query;
};
diff --git a/src/main/java/de/ids_mannheim/korap/KorapSearch.java b/src/main/java/de/ids_mannheim/korap/KorapSearch.java
index 1798afd..e8c0a6b 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapSearch.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapSearch.java
@@ -27,7 +27,8 @@
* KorapSearch implements an object for all search relevant parameters.
*/
public class KorapSearch {
- private int startIndex = 0, limit = 0;
+ private int startIndex = 0,
+ limit = 0;
private short count = 25,
countMax = 50;
private boolean cutOff = false;
@@ -35,8 +36,10 @@
private SpanQuery query;
private KorapCollection collection;
private KorapIndex index;
- private String error;
- private String warning;
+ private String error, warning;
+
+ // Timeout search after milliseconds
+ private long timeout = (long) 120_000;
private HashSet<String> fields;
@@ -45,6 +48,9 @@
public SearchContext context;
private String spanContext;
+ private long timeoutStart = Long.MIN_VALUE;
+
+
{
context = new SearchContext();
@@ -144,6 +150,10 @@
this.context.fromJSON(meta.get("context"));
// Defined resource count
+ if (meta.has("timeout"))
+ this.setTimeOut(meta.get("timeout").asLong());
+
+ // Defined resource count
if (meta.has("itemsPerResource"))
this.setItemsPerResource(meta.get("itemsPerResource").asInt());
@@ -189,6 +199,14 @@
// Empty constructor
public KorapSearch () { };
+ public long getTimeOut () {
+ return this.timeout;
+ };
+
+ public void setTimeOut (long timeout) {
+ this.timeout = timeout;
+ };
+
public String getError () {
return this.error;
};
diff --git a/src/main/java/de/ids_mannheim/korap/index/TimeOutThread.java b/src/main/java/de/ids_mannheim/korap/index/TimeOutThread.java
new file mode 100644
index 0000000..df24db1
--- /dev/null
+++ b/src/main/java/de/ids_mannheim/korap/index/TimeOutThread.java
@@ -0,0 +1,44 @@
+package de.ids_mannheim.korap.index;
+import org.apache.lucene.util.Counter;
+import java.lang.*;
+import java.lang.InterruptedException.*;
+import org.apache.lucene.util.ThreadInterruptedException;
+
+/**
+ * Create a timer thread for search time outs.
+ */
+
+// See TimeLimitingCollector
+public class TimeOutThread extends Thread {
+ private static final long resolution = 250;
+ private volatile boolean stop = false;
+ private Counter counter;
+
+ public TimeOutThread () {
+ super("TimeOutThread");
+ counter = Counter.newCounter(true);
+ };
+
+ @Override
+ public void run() {
+ while (!stop) {
+ counter.addAndGet(resolution);
+ try {
+ Thread.sleep( resolution );
+ }
+ catch (InterruptedException ie) {
+ throw new ThreadInterruptedException(ie);
+ };
+ };
+ };
+
+ // Get miliseconds
+ public long getTime () {
+ return counter.get();
+ };
+
+ // Stops the timer thread
+ public void stopTimer () {
+ stop = true;
+ };
+};
diff --git a/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java b/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
index 5ce6869..bcdd258 100644
--- a/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
+++ b/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
@@ -18,7 +18,6 @@
private String errstr, msg, version, node, listener;
private int err;
- private boolean unstaged;
private int totalResults;
private long totalTexts;
private String benchmark;
@@ -89,15 +88,6 @@
return this;
};
- public boolean getUnstaged () {
- return this.unstaged;
- };
-
- public KorapResponse setUnstaged (boolean unstaged) {
- this.unstaged = unstaged;
- return this;
- };
-
public KorapResponse setTotalTexts (long i) {
this.totalTexts = i;
return this;
@@ -107,8 +97,6 @@
return this.totalTexts;
};
-
-
public KorapResponse setTotalResults (int i) {
this.totalResults = i;
return this;
diff --git a/src/main/java/de/ids_mannheim/korap/server/Resource.java b/src/main/java/de/ids_mannheim/korap/server/Resource.java
index c92ee1d..1630bbc 100644
--- a/src/main/java/de/ids_mannheim/korap/server/Resource.java
+++ b/src/main/java/de/ids_mannheim/korap/server/Resource.java
@@ -129,11 +129,9 @@
return kresp.setError(601, "Unable to find index").toJSON();
String ID = "Unknown";
- boolean unstaged = false;
try {
FieldDocument fd = index.addDoc(uid, json);
ID = fd.getID();
- unstaged = index.getUnstaged();
}
// Set HTTP to ???
catch (IOException e) {
@@ -144,7 +142,6 @@
// Set HTTP to 200
return kresp.
setMsg("Text \"" + ID + "\" was added successfully")
- .setUnstaged(unstaged)
.toJSON();
};
@@ -178,7 +175,7 @@
};
// Set HTTP to ???
- return kresp.setMsg("Unstaged data was committed").toJSON();
+ return kresp.toJSON();
};
diff --git a/src/main/resources/index.properties b/src/main/resources/index.properties
index f1d4d58..f9ecaaf 100644
--- a/src/main/resources/index.properties
+++ b/src/main/resources/index.properties
@@ -1,3 +1,4 @@
lucene.index.version = ${project.version}
lucene.index.commit.count = 134217000
lucene.index.commit.log = log/korap.commit.log
+lucene.indexDir = /home/ndiewald/Repositories/korap/KorAP-modules/KorAP-lucene-index/sandbox/index-tanja
diff --git a/src/test/java/de/ids_mannheim/korap/server/TestResource.java b/src/test/java/de/ids_mannheim/korap/server/TestResource.java
index a5a81f9..77c87bf 100644
--- a/src/test/java/de/ids_mannheim/korap/server/TestResource.java
+++ b/src/test/java/de/ids_mannheim/korap/server/TestResource.java
@@ -80,14 +80,12 @@
assertEquals(kresp.getNode(), "milena");
assertEquals(kresp.getErr(), 0);
- assertEquals(kresp.getUnstaged(), true);
};
KorapResponse kresp = target.path("/index").
request("application/json").
post(Entity.text(""), KorapResponse.class);
assertEquals(kresp.getNode(), "milena");
- assertEquals(kresp.getMsg(), "Unstaged data was committed");
};
@Test