Merge "Added a delete method to VirtualCorpusCache."
diff --git a/Changes b/Changes
index cf39260..7bb037d 100644
--- a/Changes
+++ b/Changes
@@ -1,12 +1,13 @@
-0.60 2021-12-03
+0.60 2021-12-08
- [feature] Implemented a new cache with on disk
storage and auto-update (margaretha).
- [feature] Support for tokenized snippet output
(fixed #72; diewald)
- [cleanup] Added error messages to unsupported queries in
SpanAttributeQuery (margaretha)
- - [cleanup] Cleaned up imports (margaretha)
-
+ - [cleanup] Cleaned up imports (margaretha)
+ - Added a delete method to VirtualCorpusCache (margaretha)
+
0.59.6 2021-11-10
- [bugfix] Fixed skipping of focus spans (fixed #78; margaretha,
diewald)
diff --git a/src/main/java/de/ids_mannheim/korap/Krill.java b/src/main/java/de/ids_mannheim/korap/Krill.java
index a99624b..dc2b350 100644
--- a/src/main/java/de/ids_mannheim/korap/Krill.java
+++ b/src/main/java/de/ids_mannheim/korap/Krill.java
@@ -3,8 +3,6 @@
import java.io.IOException;
import org.apache.lucene.search.spans.SpanQuery;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
diff --git a/src/main/java/de/ids_mannheim/korap/KrillCollection.java b/src/main/java/de/ids_mannheim/korap/KrillCollection.java
index 4ec7752..14153e7 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillCollection.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillCollection.java
@@ -4,9 +4,6 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.zip.GZIPInputStream;
@@ -34,9 +31,7 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import de.ids_mannheim.korap.collection.CollectionBuilder;
-import de.ids_mannheim.korap.collection.DocBits;
import de.ids_mannheim.korap.response.Notifications;
-import de.ids_mannheim.korap.util.Fingerprinter;
import de.ids_mannheim.korap.util.KrillProperties;
import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.util.StatusCodes;
@@ -848,10 +843,7 @@
};
return str;
};
- public void storeInCache (String vcId) throws IOException, QueryException {
-
- }
-
+
@Override
public Set<String> getAllLeafFingerprints () {
return index.getAllLeafFingerprints();
diff --git a/src/main/java/de/ids_mannheim/korap/cache/VirtualCorpusCache.java b/src/main/java/de/ids_mannheim/korap/cache/VirtualCorpusCache.java
index 84cc29d..d506c86 100644
--- a/src/main/java/de/ids_mannheim/korap/cache/VirtualCorpusCache.java
+++ b/src/main/java/de/ids_mannheim/korap/cache/VirtualCorpusCache.java
@@ -17,10 +17,11 @@
import org.apache.lucene.index.LeafReaderContext;
import de.ids_mannheim.korap.IndexInfo;
+import de.ids_mannheim.korap.Krill;
import de.ids_mannheim.korap.KrillIndex;
import de.ids_mannheim.korap.collection.DocBits;
-import de.ids_mannheim.korap.collection.VirtualCorpusReferenceFilter;
-import de.ids_mannheim.korap.collection.VirtualCorpusReferenceFilter.DocBitsSupplier;
+import de.ids_mannheim.korap.collection.VirtualCorpusFilter;
+import de.ids_mannheim.korap.collection.VirtualCorpusFilter.DocBitsSupplier;
import de.ids_mannheim.korap.util.Fingerprinter;
import de.ids_mannheim.korap.util.QueryException;
@@ -64,7 +65,7 @@
if (!dir.exists()) {
dir.mkdirs();
}
-
+
String filepath = dir + "/" + leafFingerprint;
File f = new File(filepath);
if (f.exists()) {
@@ -94,7 +95,7 @@
public static void store (String vcId, KrillIndex index)
throws QueryException, IOException {
- DocBitsSupplier docBitsSupplier = new VirtualCorpusReferenceFilter(
+ DocBitsSupplier docBitsSupplier = new VirtualCorpusFilter(
vcId).new DocBitsSupplier();
String leafFingerprint;
for (LeafReaderContext context : index.reader().leaves()) {
@@ -151,12 +152,25 @@
}
}
+ public static void delete (String vcId) {
+ vcToCleanUp.remove(vcId);
+ map.remove(vcId);
+ File vc = new File(CACHE_LOCATION + "/" + vcId);
+ if (vc.exists()) {
+ for (File f : vc.listFiles()) {
+ if (f.exists()) {
+ f.delete();
+ }
+ }
+ vc.delete();
+ }
+ }
public static void reset () {
vcToCleanUp.clear();
map.clear();
- File vcCache = new File(VirtualCorpusCache.CACHE_LOCATION + "/");
+ File vcCache = new File(CACHE_LOCATION + "/");
for (File vc : vcCache.listFiles()) {
for (File f : vc.listFiles()) {
if (f.exists()) {
@@ -168,10 +182,14 @@
vcCache.delete();
}
+
/**
+ * Sets IndexInfo and checks if there is any VC to clean up. This
+ * method is called every time an index is used in {@link Krill}.
+ *
* When the VC cache knows that a leaf-fingerprint is not in the
* map of a VC, it is marked for clean up. The cached VC will be
- * cleaned up, next time a VC Reference is created.
+ * cleaned up, next time the index is used in {@link Krill}.
* see {@link #getDocBits(String, String, Supplier)}
*/
public static void setIndexInfo (IndexInfo indexInfo) {
diff --git a/src/main/java/de/ids_mannheim/korap/collection/CollectionBuilder.java b/src/main/java/de/ids_mannheim/korap/collection/CollectionBuilder.java
index 9f7e30f..166dc90 100644
--- a/src/main/java/de/ids_mannheim/korap/collection/CollectionBuilder.java
+++ b/src/main/java/de/ids_mannheim/korap/collection/CollectionBuilder.java
@@ -272,17 +272,17 @@
public class VirtualCorpusReference implements CollectionBuilder.Interface {
private boolean isNegative = false;
private String vcId;
- private VirtualCorpusReferenceFilter vcRefFilter;
+ private VirtualCorpusFilter vcFilter;
public VirtualCorpusReference (String vcId) {
this.vcId = vcId;
- vcRefFilter = new VirtualCorpusReferenceFilter(vcId);
+ vcFilter = new VirtualCorpusFilter(vcId);
// VirtualCorpusCache.setIndexInfo(indexInfo);
};
public Filter toFilter () throws QueryException {
- return vcRefFilter;
+ return vcFilter;
};
diff --git a/src/main/java/de/ids_mannheim/korap/collection/VirtualCorpusReferenceFilter.java b/src/main/java/de/ids_mannheim/korap/collection/VirtualCorpusFilter.java
similarity index 94%
rename from src/main/java/de/ids_mannheim/korap/collection/VirtualCorpusReferenceFilter.java
rename to src/main/java/de/ids_mannheim/korap/collection/VirtualCorpusFilter.java
index 2ee60bd..33c5dca 100644
--- a/src/main/java/de/ids_mannheim/korap/collection/VirtualCorpusReferenceFilter.java
+++ b/src/main/java/de/ids_mannheim/korap/collection/VirtualCorpusFilter.java
@@ -15,16 +15,16 @@
import de.ids_mannheim.korap.util.Fingerprinter;
import de.ids_mannheim.korap.util.QueryException;
-public class VirtualCorpusReferenceFilter extends Filter {
+public class VirtualCorpusFilter extends Filter {
public final static Logger log = LoggerFactory
- .getLogger(VirtualCorpusReferenceFilter.class);
+ .getLogger(VirtualCorpusFilter.class);
public static boolean DEBUG = false;
private String vcId;
private DocBitsSupplier docBitsSupplier;
- public VirtualCorpusReferenceFilter (String vcId) {
+ public VirtualCorpusFilter (String vcId) {
this.vcId = vcId;
docBitsSupplier = new DocBitsSupplier();
}
diff --git a/src/main/java/de/ids_mannheim/korap/util/Fingerprinter.java b/src/main/java/de/ids_mannheim/korap/util/Fingerprinter.java
index 59113c3..e51197c 100644
--- a/src/main/java/de/ids_mannheim/korap/util/Fingerprinter.java
+++ b/src/main/java/de/ids_mannheim/korap/util/Fingerprinter.java
@@ -29,4 +29,5 @@
return code;
}
+
}
diff --git a/src/test/java/de/ids_mannheim/korap/cache/TestVirtualCorpusCache.java b/src/test/java/de/ids_mannheim/korap/cache/TestVirtualCorpusCache.java
index bd65ab2..b456930 100644
--- a/src/test/java/de/ids_mannheim/korap/cache/TestVirtualCorpusCache.java
+++ b/src/test/java/de/ids_mannheim/korap/cache/TestVirtualCorpusCache.java
@@ -64,7 +64,8 @@
Map<String, DocBits> docIdMap = VirtualCorpusCache.retrieve(vcId);
assertEquals(2, docIdMap.size());
- VirtualCorpusCache.reset();
+ VirtualCorpusCache.delete(vcId);
+ assertFalse(VirtualCorpusCache.contains(vcId));
}
@@ -81,12 +82,14 @@
Map<String, DocBits> vc1 = VirtualCorpusCache.retrieve(vcId);
assertNotNull(vc1);
- VirtualCorpusCache.reset();
+ VirtualCorpusCache.delete(vcId);
+ assertFalse(VirtualCorpusCache.contains(vcId));
}
@Test
public void testUpdateCachedVC () throws IOException {
+ String vcId = "named-vc1";
// VC cache will be marked for cleaning up
// because of storing a new VC
KrillIndex ki = createIndex();
@@ -95,7 +98,7 @@
assertEquals(27, result.getTotalResults());
assertEquals(2,
- VirtualCorpusCache.map.get("named-vc1").keySet().size());
+ VirtualCorpusCache.map.get(vcId).keySet().size());
ki.delDoc(2);
ki.commit();
@@ -110,7 +113,7 @@
// should have the same size. But the fingerprints should be
// different from before the 1st cleaning up
assertEquals(2,
- VirtualCorpusCache.map.get("named-vc1").keySet().size());
+ VirtualCorpusCache.map.get(vcId).keySet().size());
// VC cache will be cleaned up for the 2nd time
// resulting the same leaf-fingerprints
@@ -119,11 +122,12 @@
assertEquals(17, result.getTotalResults());
assertEquals(2,
- VirtualCorpusCache.map.get("named-vc1").keySet().size());
+ VirtualCorpusCache.map.get(vcId).keySet().size());
ki.close();
- VirtualCorpusCache.reset();
+ VirtualCorpusCache.delete(vcId);
+ assertFalse(VirtualCorpusCache.contains(vcId));
}