Updated VC cache.
Change-Id: Id4663a9b51b8324669883df924370a9932af6076
diff --git a/full/src/main/java/de/ids_mannheim/de/init/VCLoaderImpl.java b/full/src/main/java/de/ids_mannheim/de/init/VCLoaderImpl.java
index 113a6cc..6c00e4c 100644
--- a/full/src/main/java/de/ids_mannheim/de/init/VCLoaderImpl.java
+++ b/full/src/main/java/de/ids_mannheim/de/init/VCLoaderImpl.java
@@ -12,7 +12,7 @@
@Override
public void recachePredefinedVC () {
- KrillCollection.cache.removeAll();
+// KrillCollection.cache.removeAll();
Thread t = new Thread(vcLoader);
t.start();
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java b/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
index ac782b6..38cc991 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
@@ -17,16 +17,27 @@
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
-import de.ids_mannheim.korap.KrillCollection;
+import de.ids_mannheim.korap.cache.VirtualCorpusCache;
import de.ids_mannheim.korap.constant.QueryType;
import de.ids_mannheim.korap.constant.ResourceType;
-import de.ids_mannheim.korap.entity.QueryDO;
import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.exceptions.StatusCodes;
import de.ids_mannheim.korap.service.QueryService;
import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.web.SearchKrill;
-/** Loads predefined virtual corpora at server start up and cache them.
+/**
+ * <p>Loads predefined virtual corpora at server start up and caches
+ * them, if the VC have not been cached before. If there are changes
+ * in the index, the cache will be updated.
+ * </p>
+ *
+ * <p>
+ * All predefined VC are set as SYSTEM VC. The filenames are used as
+ * VC names. Acceptable file extensions are .jsonld.gz or .jsonld. The
+ * VC should be located at the folder indicated by <em>krill.namedVC</em>
+ * specified in kustvakt.conf.
+ * </p>
*
* @author margaretha
*
@@ -48,19 +59,27 @@
try {
loadVCToCache();
}
- catch (IOException | QueryException | KustvaktException e) {
+ catch (IOException | QueryException e) {
// e.printStackTrace();
throw new RuntimeErrorException(new Error(e.getMessage(), e.getCause()));
}
}
+ /** Used for testing
+ *
+ * @param filename
+ * @param filePath
+ * @throws IOException
+ * @throws QueryException
+ * @throws KustvaktException
+ */
public void loadVCToCache (String filename, String filePath)
throws IOException, QueryException, KustvaktException {
InputStream is = NamedVCLoader.class.getResourceAsStream(filePath);
String json = IOUtils.toString(is, "utf-8");
if (json != null) {
- cacheVC(json, filename);
+ cacheVC(filename,json);
vcService.storeQuery("system",filename, ResourceType.SYSTEM,
QueryType.VIRTUAL_CORPUS, json, null, null, null, true,
"system", null, null);
@@ -68,7 +87,7 @@
}
public void loadVCToCache ()
- throws IOException, QueryException, KustvaktException {
+ throws IOException, QueryException {
String dir = config.getNamedVCPath();
if (dir.isEmpty()) return;
@@ -90,31 +109,14 @@
filename = strArr[0];
String json = strArr[1];
if (json != null) {
- cacheVC(json, filename);
- try {
- QueryDO vc = vcService.searchQueryByName("system",
- filename, "system", QueryType.VIRTUAL_CORPUS);
- if (vc != null) {
- if (DEBUG) {
- jlog.debug("Delete existing vc: " + filename);
- }
- vcService.deleteQueryByName("system", vc.getName(),
- vc.getCreatedBy(), QueryType.VIRTUAL_CORPUS);
- }
- }
- catch (KustvaktException e) {
- // ignore
- if (DEBUG) jlog.debug(e);
- }
- vcService.storeQuery("system",filename, ResourceType.SYSTEM,
- QueryType.VIRTUAL_CORPUS, json, null, null, null, true,
- "system", null, null);
+ cacheVC(filename,json);
+ storeVCinDB(filename, json);
}
}
}
private String[] readFile (File file, String filename)
- throws IOException, KustvaktException {
+ throws IOException {
String json = null;
long start = System.currentTimeMillis();
if (filename.endsWith(".jsonld")) {
@@ -142,25 +144,54 @@
return new String[] { filename, json };
}
- private void cacheVC (String json, String filename)
+ /**
+ * Caches the given VC if the VC is not found in cache and updates
+ * the VC if it exists and there are changes in the index.
+ *
+ * @param vcId
+ * vc-name
+ * @param koralQuery
+ * @throws IOException
+ * @throws QueryException
+ */
+ private void cacheVC (String vcId, String koralQuery)
throws IOException, QueryException {
- config.setVcInCaching(filename);
+ config.setVcInCaching(vcId);
+ jlog.info("Storing {} in cache ", vcId);
long start, end;
start = System.currentTimeMillis();
-
- KrillCollection collection = new KrillCollection(json);
- collection.setIndex(searchKrill.getIndex());
-
- jlog.info("Storing {} in cache ", filename);
- if (collection != null) {
- collection.storeInCache(filename);
- }
+ VirtualCorpusCache.store(vcId, searchKrill.getIndex());
end = System.currentTimeMillis();
- jlog.info("{} Caching duration: {}", filename, (end - start));
- if (DEBUG) {
- jlog.debug("memory cache: "
- + KrillCollection.cache.calculateInMemorySize());
- }
+ jlog.info("{} Caching duration: {}", vcId, (end - start));
config.setVcInCaching("");
}
+
+ /** Stores the VC if it doesn't exist in the database.
+ *
+ * @param vcId
+ * @param koralQuery
+ */
+ private void storeVCinDB (String vcId, String koralQuery) {
+ try {
+ vcService.searchQueryByName("system", vcId, "system",
+ QueryType.VIRTUAL_CORPUS);
+ }
+ catch (KustvaktException e) {
+ if (e.getStatusCode() == StatusCodes.NO_RESOURCE_FOUND) {
+ try {
+ jlog.info("Storing {} in database ", vcId);
+ vcService.storeQuery("system", vcId, ResourceType.SYSTEM,
+ QueryType.VIRTUAL_CORPUS, koralQuery, null, null,
+ null, true, "system", null, null);
+ }
+ catch (KustvaktException e1) {
+ throw new RuntimeException(e1);
+ }
+ }
+ else {
+ throw new RuntimeException(e);
+ }
+ }
+
+ }
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/service/QueryService.java b/full/src/main/java/de/ids_mannheim/korap/service/QueryService.java
index 7e89803..68216d5 100644
--- a/full/src/main/java/de/ids_mannheim/korap/service/QueryService.java
+++ b/full/src/main/java/de/ids_mannheim/korap/service/QueryService.java
@@ -17,11 +17,11 @@
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
-import de.ids_mannheim.korap.KrillCollection;
+import de.ids_mannheim.korap.cache.VirtualCorpusCache;
import de.ids_mannheim.korap.config.FullConfiguration;
import de.ids_mannheim.korap.constant.GroupMemberStatus;
-import de.ids_mannheim.korap.constant.QueryType;
import de.ids_mannheim.korap.constant.QueryAccessStatus;
+import de.ids_mannheim.korap.constant.QueryType;
import de.ids_mannheim.korap.constant.ResourceType;
import de.ids_mannheim.korap.dao.AdminDao;
import de.ids_mannheim.korap.dao.QueryAccessDao;
@@ -30,10 +30,10 @@
import de.ids_mannheim.korap.dto.QueryDto;
import de.ids_mannheim.korap.dto.converter.QueryAccessConverter;
import de.ids_mannheim.korap.dto.converter.QueryConverter;
+import de.ids_mannheim.korap.entity.QueryAccess;
+import de.ids_mannheim.korap.entity.QueryDO;
import de.ids_mannheim.korap.entity.UserGroup;
import de.ids_mannheim.korap.entity.UserGroupMember;
-import de.ids_mannheim.korap.entity.QueryDO;
-import de.ids_mannheim.korap.entity.QueryAccess;
import de.ids_mannheim.korap.exceptions.KustvaktException;
import de.ids_mannheim.korap.exceptions.StatusCodes;
import de.ids_mannheim.korap.query.serialize.QuerySerializer;
@@ -188,8 +188,8 @@
access.getUserGroup().getId(), "system");
}
if (type.equals(QueryType.VIRTUAL_CORPUS)
- && KrillCollection.cache.get(query.getName()) != null) {
- KrillCollection.cache.remove(query.getName());
+ && VirtualCorpusCache.contains(queryName)) {
+ VirtualCorpusCache.delete(queryName);
}
queryDao.deleteQuery(query);
}