Added total result cache (close #599)
Change-Id: Iacf748bd0f43597c65bf1bf0511dd69e9467a086
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/KustvaktCacheable.java b/full/src/main/java/de/ids_mannheim/korap/config/KustvaktCacheable.java
index b22cb70..a66486f 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/KustvaktCacheable.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/KustvaktCacheable.java
@@ -1,6 +1,7 @@
package de.ids_mannheim.korap.config;
import java.io.InputStream;
+import java.util.List;
import java.util.Map;
import de.ids_mannheim.korap.utils.ServiceInfo;
@@ -115,4 +116,9 @@
Cache cache = getCache(name);
return cache.getAll(cache.getKeysWithExpiryCheck());
}
+
+ public List getKeysWithExpiryCheck () {
+ Cache cache = getCache(name);
+ return cache.getKeysWithExpiryCheck();
+ }
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java b/full/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
index 96b5f62..7cfdcac 100644
--- a/full/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
+++ b/full/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
@@ -28,6 +28,7 @@
//import de.ids_mannheim.de.init.VCLoader;
import de.ids_mannheim.korap.authentication.AuthenticationManager;
+import de.ids_mannheim.korap.config.KustvaktCacheable;
import de.ids_mannheim.korap.config.KustvaktConfiguration;
import de.ids_mannheim.korap.exceptions.KustvaktException;
import de.ids_mannheim.korap.exceptions.StatusCodes;
@@ -43,6 +44,13 @@
@Service
public class SearchService extends BasicService{
+
+ public class TotalResultCache extends KustvaktCacheable{
+
+ public TotalResultCache () {
+ super("total_results","key:hashedKoralQuery");
+ }
+ }
private static final boolean DEBUG = false;
@@ -62,11 +70,15 @@
private SearchNetworkEndpoint searchNetwork;
private ClientsHandler graphDBhandler;
+
+ private TotalResultCache totalResultCache;
@PostConstruct
private void doPostConstruct () {
UriBuilder builder = UriBuilder.fromUri("http://10.0.10.13").port(9997);
this.graphDBhandler = new ClientsHandler(builder.build());
+
+ totalResultCache = new TotalResultCache();
}
public String getKrillVersion () {
@@ -194,6 +206,12 @@
jlog.debug("the serialized query " + query);
}
+ int hashedKoralQuery = createTotalResultCacheKey(query);
+ boolean hasCutOff = hasCutOff(query);
+ if (!hasCutOff) {
+ query = precheckTotalResultCache(hashedKoralQuery,query);
+ }
+
KustvaktConfiguration.BACKENDS searchEngine = this.config.chooseBackend(engine);
String result;
if (searchEngine.equals(KustvaktConfiguration.BACKENDS.NEO4J)) {
@@ -206,9 +224,79 @@
result = searchKrill.search(query);
}
// jlog.debug("Query result: " + result);
+
+ result = afterCheckTotalResultCache(hashedKoralQuery,result);
+ if (!hasCutOff) {
+ result = removeCutOff(result);
+ }
return result;
}
+
+ private String removeCutOff (String result) throws KustvaktException {
+ ObjectNode resultNode = (ObjectNode) JsonUtils.readTree(result);
+ ObjectNode meta = (ObjectNode) resultNode.at("/meta");
+ meta.remove("cutOff");
+ return resultNode.toString();
+ }
+
+ public int createTotalResultCacheKey (String query) throws KustvaktException {
+ ObjectNode queryNode = (ObjectNode) JsonUtils.readTree(query);
+ queryNode.remove("meta");
+ return queryNode.hashCode();
+ }
+
+ private String afterCheckTotalResultCache (int hashedKoralQuery,
+ String result) throws KustvaktException {
+
+ String totalResults =
+ (String) totalResultCache.getCacheValue(hashedKoralQuery);
+ if (totalResults != null) {
+ ObjectNode queryNode = (ObjectNode) JsonUtils.readTree(result);
+ ObjectNode meta = (ObjectNode) queryNode.at("/meta");
+ if (meta.isMissingNode()) {
+ queryNode.put("totalResults", totalResults);
+ }
+ else {
+ meta.put("totalResults", totalResults);
+ }
+ result = queryNode.toString();
+ }
+ else {
+ JsonNode node = JsonUtils.readTree(result);
+ totalResults = node.at("/meta/totalResults").asText();
+ if (totalResults != null &&
+ !totalResults.isEmpty() &&
+ Integer.parseInt(totalResults) > 0)
+ totalResultCache.storeInCache(hashedKoralQuery, totalResults);
+ }
+ return result;
+ }
+
+ public String precheckTotalResultCache (int hashedKoralQuery, String query)
+ throws KustvaktException {
+ String totalResults =
+ (String) totalResultCache.getCacheValue(hashedKoralQuery);
+ if (totalResults != null) {
+ // add cutoff
+ ObjectNode queryNode = (ObjectNode) JsonUtils.readTree(query);
+ ObjectNode meta = (ObjectNode) queryNode.at("/meta");
+ meta.put("cutOff", "true");
+ query = queryNode.toString();
+ }
+ return query;
+ }
+
+ private boolean hasCutOff (String query) throws KustvaktException {
+ JsonNode queryNode = JsonUtils.readTree(query);
+ JsonNode cutOff = queryNode.at("/meta/cutOff");
+ if (cutOff.isMissingNode()) {
+ return false;
+ }
+ else {
+ return true;
+ }
+ }
/**
* Pipes are service URLs for modifying KoralQuery. A POST request
@@ -490,4 +578,8 @@
public String getIndexFingerprint () {
return searchKrill.getIndexFingerprint();
}
+
+ public TotalResultCache getTotalResultCache () {
+ return totalResultCache;
+ }
}
diff --git a/full/src/main/resources/ehcache.xml b/full/src/main/resources/ehcache.xml
index 12bd50b..22f2f55 100644
--- a/full/src/main/resources/ehcache.xml
+++ b/full/src/main/resources/ehcache.xml
@@ -30,5 +30,14 @@
diskExpiryThreadIntervalSeconds = "120" >
<persistence strategy="localTempSwap"/>
<sizeOfPolicy maxDepth="3000" maxDepthExceededBehavior="abort" />
- </cache> -->
+ -->
+
+ <cache name="total_results"
+ timeToIdleSeconds="3600"
+ timeToLiveSeconds="15000"
+ eternal='false'
+ memoryStoreEvictionPolicy="LRU"
+ overflowToDisk='false'
+ maxEntriesLocalHeap="500"
+ />
</ehcache>