Added total result cache (close #599)
Change-Id: Iacf748bd0f43597c65bf1bf0511dd69e9467a086
diff --git a/full/Changes b/full/Changes
index 7b459df..caa143e 100644
--- a/full/Changes
+++ b/full/Changes
@@ -4,6 +4,7 @@
- Removed old database configurations (#612)
- Removed old tests
- Removed unnecessary caches and methods in the authentication interface
+- Added total result cache (#599)
# version 0.70.1
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/KustvaktCacheable.java b/full/src/main/java/de/ids_mannheim/korap/config/KustvaktCacheable.java
index b22cb70..a66486f 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/KustvaktCacheable.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/KustvaktCacheable.java
@@ -1,6 +1,7 @@
package de.ids_mannheim.korap.config;
import java.io.InputStream;
+import java.util.List;
import java.util.Map;
import de.ids_mannheim.korap.utils.ServiceInfo;
@@ -115,4 +116,9 @@
Cache cache = getCache(name);
return cache.getAll(cache.getKeysWithExpiryCheck());
}
+
+ public List getKeysWithExpiryCheck () {
+ Cache cache = getCache(name);
+ return cache.getKeysWithExpiryCheck();
+ }
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java b/full/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
index 96b5f62..7cfdcac 100644
--- a/full/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
+++ b/full/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
@@ -28,6 +28,7 @@
//import de.ids_mannheim.de.init.VCLoader;
import de.ids_mannheim.korap.authentication.AuthenticationManager;
+import de.ids_mannheim.korap.config.KustvaktCacheable;
import de.ids_mannheim.korap.config.KustvaktConfiguration;
import de.ids_mannheim.korap.exceptions.KustvaktException;
import de.ids_mannheim.korap.exceptions.StatusCodes;
@@ -43,6 +44,13 @@
@Service
public class SearchService extends BasicService{
+
+ public class TotalResultCache extends KustvaktCacheable{
+
+ public TotalResultCache () {
+ super("total_results","key:hashedKoralQuery");
+ }
+ }
private static final boolean DEBUG = false;
@@ -62,11 +70,15 @@
private SearchNetworkEndpoint searchNetwork;
private ClientsHandler graphDBhandler;
+
+ private TotalResultCache totalResultCache;
@PostConstruct
private void doPostConstruct () {
UriBuilder builder = UriBuilder.fromUri("http://10.0.10.13").port(9997);
this.graphDBhandler = new ClientsHandler(builder.build());
+
+ totalResultCache = new TotalResultCache();
}
public String getKrillVersion () {
@@ -194,6 +206,12 @@
jlog.debug("the serialized query " + query);
}
+ int hashedKoralQuery = createTotalResultCacheKey(query);
+ boolean hasCutOff = hasCutOff(query);
+ if (!hasCutOff) {
+ query = precheckTotalResultCache(hashedKoralQuery,query);
+ }
+
KustvaktConfiguration.BACKENDS searchEngine = this.config.chooseBackend(engine);
String result;
if (searchEngine.equals(KustvaktConfiguration.BACKENDS.NEO4J)) {
@@ -206,9 +224,79 @@
result = searchKrill.search(query);
}
// jlog.debug("Query result: " + result);
+
+ result = afterCheckTotalResultCache(hashedKoralQuery,result);
+ if (!hasCutOff) {
+ result = removeCutOff(result);
+ }
return result;
}
+
+ private String removeCutOff (String result) throws KustvaktException {
+ ObjectNode resultNode = (ObjectNode) JsonUtils.readTree(result);
+ ObjectNode meta = (ObjectNode) resultNode.at("/meta");
+ meta.remove("cutOff");
+ return resultNode.toString();
+ }
+
+ public int createTotalResultCacheKey (String query) throws KustvaktException {
+ ObjectNode queryNode = (ObjectNode) JsonUtils.readTree(query);
+ queryNode.remove("meta");
+ return queryNode.hashCode();
+ }
+
+ private String afterCheckTotalResultCache (int hashedKoralQuery,
+ String result) throws KustvaktException {
+
+ String totalResults =
+ (String) totalResultCache.getCacheValue(hashedKoralQuery);
+ if (totalResults != null) {
+ ObjectNode queryNode = (ObjectNode) JsonUtils.readTree(result);
+ ObjectNode meta = (ObjectNode) queryNode.at("/meta");
+ if (meta.isMissingNode()) {
+ queryNode.put("totalResults", totalResults);
+ }
+ else {
+ meta.put("totalResults", totalResults);
+ }
+ result = queryNode.toString();
+ }
+ else {
+ JsonNode node = JsonUtils.readTree(result);
+ totalResults = node.at("/meta/totalResults").asText();
+ if (totalResults != null &&
+ !totalResults.isEmpty() &&
+ Integer.parseInt(totalResults) > 0)
+ totalResultCache.storeInCache(hashedKoralQuery, totalResults);
+ }
+ return result;
+ }
+
+ public String precheckTotalResultCache (int hashedKoralQuery, String query)
+ throws KustvaktException {
+ String totalResults =
+ (String) totalResultCache.getCacheValue(hashedKoralQuery);
+ if (totalResults != null) {
+ // add cutoff
+ ObjectNode queryNode = (ObjectNode) JsonUtils.readTree(query);
+ ObjectNode meta = (ObjectNode) queryNode.at("/meta");
+ meta.put("cutOff", "true");
+ query = queryNode.toString();
+ }
+ return query;
+ }
+
+ private boolean hasCutOff (String query) throws KustvaktException {
+ JsonNode queryNode = JsonUtils.readTree(query);
+ JsonNode cutOff = queryNode.at("/meta/cutOff");
+ if (cutOff.isMissingNode()) {
+ return false;
+ }
+ else {
+ return true;
+ }
+ }
/**
* Pipes are service URLs for modifying KoralQuery. A POST request
@@ -490,4 +578,8 @@
public String getIndexFingerprint () {
return searchKrill.getIndexFingerprint();
}
+
+ public TotalResultCache getTotalResultCache () {
+ return totalResultCache;
+ }
}
diff --git a/full/src/main/resources/ehcache.xml b/full/src/main/resources/ehcache.xml
index 12bd50b..22f2f55 100644
--- a/full/src/main/resources/ehcache.xml
+++ b/full/src/main/resources/ehcache.xml
@@ -30,5 +30,14 @@
diskExpiryThreadIntervalSeconds = "120" >
<persistence strategy="localTempSwap"/>
<sizeOfPolicy maxDepth="3000" maxDepthExceededBehavior="abort" />
- </cache> -->
+ -->
+
+ <cache name="total_results"
+ timeToIdleSeconds="3600"
+ timeToLiveSeconds="15000"
+ eternal='false'
+ memoryStoreEvictionPolicy="LRU"
+ overflowToDisk='false'
+ maxEntriesLocalHeap="500"
+ />
</ehcache>
diff --git a/full/src/test/java/de/ids_mannheim/korap/cache/TotalResultTest.java b/full/src/test/java/de/ids_mannheim/korap/cache/TotalResultTest.java
new file mode 100644
index 0000000..b958245
--- /dev/null
+++ b/full/src/test/java/de/ids_mannheim/korap/cache/TotalResultTest.java
@@ -0,0 +1,114 @@
+package de.ids_mannheim.korap.cache;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import de.ids_mannheim.korap.config.SpringJerseyTest;
+import de.ids_mannheim.korap.core.service.SearchService;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+public class TotalResultTest extends SpringJerseyTest {
+
+ @Autowired
+ private SearchService searchService;
+
+ @Test
+ public void testSearchWithPaging () throws KustvaktException {
+
+ assertEquals(0, searchService.getTotalResultCache()
+ .getAllCacheElements().size());
+
+ Response response = target().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=die]").queryParam("ql", "poliqarp")
+ .queryParam("page", "1").request().get();
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ String entity = response.readEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+ int totalResults = node.at("/meta/totalResults").asInt();
+
+ assertEquals(1, searchService.getTotalResultCache()
+ .getAllCacheElements().size());
+
+ response = target().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=die]").queryParam("ql", "poliqarp")
+ .queryParam("page", "2").request().get();
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ entity = response.readEntity(String.class);
+ node = JsonUtils.readTree(entity);
+ assertEquals(totalResults, node.at("/meta/totalResults").asInt());
+
+ assertEquals(1, searchService.getTotalResultCache()
+ .getAllCacheElements().size());
+
+ assertTrue(node.at("/meta/cutOff").isMissingNode());
+
+ testSearchWithCutOff();
+ }
+
+ @Test
+ public void testSearchWithCutOffTrue () throws KustvaktException {
+
+ int cacheSize = searchService.getTotalResultCache()
+ .getAllCacheElements().size();
+
+ Response response = target().path(API_VERSION).path("search")
+ .queryParam("q", "ich").queryParam("ql", "poliqarp")
+ .queryParam("page", "2")
+ .queryParam("cutoff", "true")
+ .request().get();
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+ String query = "{\"meta\":{\"startPage\":2,\"tokens\":false,\"cutOff\":"
+ + "true,\"snippets\":true,\"timeout\":10000},\"query\":{\"@type\":"
+ + "\"koral:token\",\"wrap\":{\"@type\":\"koral:term\",\"match\":"
+ + "\"match:eq\",\"layer\":\"orth\",\"key\":\"ich\",\"foundry\":"
+ + "\"opennlp\",\"rewrites\":[{\"@type\":\"koral:rewrite\",\"src\":"
+ + "\"Kustvakt\",\"operation\":\"operation:injection\",\"scope\":"
+ + "\"foundry\"}]}},\"@context\":\"http://korap.ids-mannheim.de/ns"
+ + "/koral/0.3/context.jsonld\",\"collection\":{\"@type\":\"koral:"
+ + "doc\",\"match\":\"match:eq\",\"type\":\"type:regex\",\"value\":"
+ + "\"CC-BY.*\",\"key\":\"availability\",\"rewrites\":[{\"@type\":"
+ + "\"koral:rewrite\",\"src\":\"Kustvakt\",\"operation\":\"operation:"
+ + "insertion\",\"scope\":\"availability(FREE)\"}]}}";
+
+ int cacheKey = searchService.createTotalResultCacheKey(query);
+ assertEquals(null, searchService.getTotalResultCache()
+ .getCacheValue(cacheKey));
+
+ assertEquals(cacheSize, searchService.getTotalResultCache()
+ .getAllCacheElements().size());
+ }
+
+ private void testSearchWithCutOff () throws KustvaktException {
+
+ Response response = target().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=die]").queryParam("ql", "poliqarp")
+ .queryParam("page", "3").queryParam("cutoff", "false").request()
+ .get();
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ String entity = response.readEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+
+ assertTrue(node.at("/meta/cutOff").isMissingNode());
+
+ response = target().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=die]").queryParam("ql", "poliqarp")
+ .queryParam("page", "4").queryParam("cutoff", "true").request()
+ .get();
+
+ entity = response.readEntity(String.class);
+ node = JsonUtils.readTree(entity);
+
+ assertTrue(node.at("/meta/cutOff").asBoolean());
+ }
+}
diff --git a/full/src/test/java/de/ids_mannheim/korap/web/controller/SearchControllerTest.java b/full/src/test/java/de/ids_mannheim/korap/web/controller/SearchControllerTest.java
index 37b2b6a..8bdee32 100644
--- a/full/src/test/java/de/ids_mannheim/korap/web/controller/SearchControllerTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/web/controller/SearchControllerTest.java
@@ -166,7 +166,7 @@
@Test
public void testSearchQueryWithMeta () throws KustvaktException {
Response response = target().path(API_VERSION).path("search")
- .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("q", "[orth=Bachelor]").queryParam("ql", "poliqarp")
.queryParam("cutoff", "true").queryParam("count", "5")
.queryParam("page", "1").queryParam("context", "40-t,30-t")
.request()