Test LeafFingerprints after index change.
Change-Id: I799b453311a453c6c5cb65db3eb62114f7c0ea54
diff --git a/src/test/java/de/ids_mannheim/korap/TestIndexer.java b/src/test/java/de/ids_mannheim/korap/TestIndexer.java
index 9f0390c..246668b 100644
--- a/src/test/java/de/ids_mannheim/korap/TestIndexer.java
+++ b/src/test/java/de/ids_mannheim/korap/TestIndexer.java
@@ -13,6 +13,9 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
import java.util.zip.GZIPOutputStream;
import org.junit.After;
@@ -329,6 +332,74 @@
}
+ /** Test two identical index and check that the leaf readers return
+ * the same fingerprints. After deleting a document, check again if the
+ * leaf fingerprints are identical.
+ *
+ * This is to make sure that cached leaf-readers are can be copied across
+ * multiple identical indexes with the same index updates, to avoid
+ * the overhead of re-caching leaf readers for each index.
+ */
+ @Test
+ public void testLeafReader () throws IOException {
+ String outputDir1 = createWikiIndex("wiki1",1,3);
+ outputDir1 = createWikiIndex("wiki1",4,5);
+ KrillIndex ki1 = new KrillIndex(Paths.get(outputDir1));
+ assertEquals(5, ki1.numberOf("documents"));
+
+ List<String> fp1 = new ArrayList<>(ki1.getAllLeafFingerprints());
+ Collections.sort(fp1);
+
+ delWikiDoc(outputDir1, "WPD_AAA.00001");
+
+ ki1 = new KrillIndex(Paths.get(outputDir1));
+ assertEquals(4, ki1.numberOf("documents"));
+ List<String> fp1a = new ArrayList<>(ki1.getAllLeafFingerprints());
+ Collections.sort(fp1a);
+
+ // 2nd index identical to the 1st index
+ String outputDir2 = createWikiIndex("wiki2", 1, 3);
+ outputDir2 = createWikiIndex("wiki2", 4, 5);
+ KrillIndex ki2 = new KrillIndex(Paths.get(outputDir2));
+ assertEquals(5, ki2.numberOf("documents"));
+
+ List<String> fp2 = new ArrayList<>(ki2.getAllLeafFingerprints());
+ Collections.sort(fp2);
+ assertEquals(fp1, fp2);
+
+ delWikiDoc(outputDir2, "WPD_AAA.00001");
+
+ ki2 = new KrillIndex(Paths.get(outputDir2));
+ assertEquals(4, ki2.numberOf("documents"));
+ List<String> fp2a = new ArrayList<>(ki1.getAllLeafFingerprints());
+ Collections.sort(fp2a);
+ assertEquals(fp1a, fp2a);
+ ki1.close();
+ ki2.close();
+ }
+
+ private void delWikiDoc(String outputDir, String textSigle) throws IOException {
+ outputStream.reset();
+ Indexer.main(new String[] { "-c", "src/test/resources/krill.properties",
+ "-o", outputDir, "-D", "ID", textSigle });
+ //assertTrue(outputStream.toString()
+ // .startsWith("Deleted documents where textSigle="+textSigle+"."));
+ }
+
+ private String createWikiIndex (String input, int startId, int endId) throws IOException {
+ Path inputDir = Files.createTempDirectory(tempBaseDirectory.toPath(),
+ input);
+ for (int i = startId; i <= endId; i++) {
+ Path jsonPath = Paths.get("src/test/resources/wiki/0000" + i + ".json");
+ Path gzPath = inputDir.resolve("0000" + i + ".json.gz");
+ gzipFile(jsonPath, gzPath);
+ }
+ String outputDir = getTestOutputPath(input + "-index");
+ Indexer.main(new String[] { "-c", "src/test/resources/krill.properties",
+ "-i", inputDir.toString(), "-o", outputDir });
+ return outputDir;
+ }
+
@Before
public void setOutputStream () {
System.setOut(new PrintStream(outputStream));
diff --git a/src/test/java/de/ids_mannheim/korap/cache/TestVirtualCorpusCache.java b/src/test/java/de/ids_mannheim/korap/cache/TestVirtualCorpusCache.java
index 1d8f462..8fced53 100644
--- a/src/test/java/de/ids_mannheim/korap/cache/TestVirtualCorpusCache.java
+++ b/src/test/java/de/ids_mannheim/korap/cache/TestVirtualCorpusCache.java
@@ -2,6 +2,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -9,6 +10,7 @@
import java.io.IOException;
import java.io.InputStream;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.io.IOUtils;
import org.junit.Test;
@@ -125,9 +127,14 @@
assertEquals(27, result.getTotalResults());
assertEquals(2, VirtualCorpusCache.map.get(vcId).keySet().size());
-
+ Set<String> fp1 = ki.getAllLeafFingerprints();
+
ki.delDoc(2);
ki.commit();
+
+ Set<String> fp2 = ki.getAllLeafFingerprints();
+
+ assertNotEquals(fp1, fp2);
// VC cache will be marked for cleaning up again
// because of index change.