Merge "Add GitHub release workflow for version tag pushes"
diff --git a/Changes b/Changes
index 3cf53bb..a5366f3 100644
--- a/Changes
+++ b/Changes
@@ -1,9 +1,16 @@
+0.65.1 2026-06-16
+ - [feature] Prevent indexation of documents without
+ a token stream (diewald; AI-assisted Claude Opus 4.6)
+ - [bugfix] Fix "force" in commit method
+ (diewald; AI-assisted Claude Opus 4.6)
+ - [tool] Add standalone MetaExporter tool
+ (kupietz; AI-assistend Claude Opus 4.8)
+ - [bugfix] Fix cross-doc leakage in nested WithinSpans
+ (diewald; AI-assisted Claude Opus 4.6)
+
0.65 2026-05-12
-includes all changes in 0.64.7
-
-0.64.7 2026-05-07
- [bugfix] Keep highlights that extend beyond a cut match
- (diewald; fixes #177; diewald; AI-assisted Claude Opus 4.6)
+ (diewald; fixes #177; AI-assisted Claude Opus 4.6)
- [bugfix] Correctly handle foundry and layer in attribute groups
(diewald; AI-assisted Claude Opus 4.6)
- [bugfix] Support regular expressions in attribute queries
diff --git a/pom.xml b/pom.xml
index ae4d153..3d8657c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -29,13 +29,28 @@
for example:
$ java -jar target/Krill-Indexer.jar -c src/test/resources/krill.properties
- -i src/test/resources/bzk -o index/
-
+ -i src/test/resources/bzk -o index/
+
+ ** MetaExporter
+ # after packaging (see above)
+ # dumps the stored per-document metadata of an existing index,
+ # without using the web service API.
+
+ $ java -jar target/Krill-MetaExporter.jar -i [index directory]
+ -f [comma-separated fields] [format tsv|csv|json] [-o output file]
+
+ for example (list fields, then dump text sigles for diffing instances):
+
+ $ java -jar target/Krill-MetaExporter.jar -i index/ (option) list-fields
+ $ java -jar target/Krill-MetaExporter.jar -i index/ -f textSigle (option) no-header
+
+ (full option names use a double-dash prefix; see the MetaExporter Javadoc)
+
-->
<groupId>de.ids-mannheim.korap.krill</groupId>
<artifactId>Krill</artifactId>
- <version>0.65</version>
+ <version>0.65.1</version>
<packaging>jar</packaging>
<name>Krill</name>
@@ -62,7 +77,7 @@
<java.version>21</java.version>
<jersey.version>4.0.2</jersey.version>
<log4j.version>2.26.0</log4j.version>
- <jackson.version>2.21.4</jackson.version>
+ <jackson.version>2.22.0</jackson.version>
<jackson.mainversion>2.22</jackson.mainversion>
<lucene.version>5.0.0</lucene.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -122,14 +137,14 @@
<dependency>
<groupId>org.xerial</groupId>
<artifactId>sqlite-jdbc</artifactId>
- <version>3.53.1.0</version>
+ <version>3.53.2.0</version>
</dependency>
<!-- Database Connection Pool Manager -->
<dependency>
<groupId>com.mchange</groupId>
<artifactId>c3p0</artifactId>
- <version>0.13.0</version>
+ <version>0.14.1</version>
</dependency>
<!-- Lucene core dependency -->
@@ -278,6 +293,23 @@
</configuration>
</execution>
<execution>
+ <id>metaexporter</id>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <createDependencyReducedPom>false</createDependencyReducedPom>
+ <transformers>
+ <transformer
+ implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>de.ids_mannheim.korap.index.MetaExporter</mainClass>
+ </transformer>
+ </transformers>
+ <finalName>${project.artifactId}-MetaExporter</finalName>
+ </configuration>
+ </execution>
+ <execution>
<id>server</id>
<phase>package</phase>
<goals>
diff --git a/src/main/java/de/ids_mannheim/korap/KrillIndex.java b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
index 0bcd1cd..892bc05 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
@@ -444,7 +444,7 @@
*/
public void commit (boolean force) throws IOException {
// There is something to commit
- if (commitCounter > 0 || !force)
+ if (commitCounter > 0 || force)
this.commit();
};
@@ -496,6 +496,15 @@
if (doc == null)
return doc;
+ if (!hasTokenField(doc)) {
+ log.error(
+ "Rejecting upsert for document '{}': no token stream - "
+ + "existing document (if any) will not be removed",
+ doc.getTextSigle()
+ );
+ return doc;
+ }
+
// Create a filter based on the corpusID and the docID
String textSigle = doc.getTextSigle();
KrillDate current = new KrillDate(LocalDate.now());
@@ -608,6 +617,26 @@
/**
+ * Check if a FieldDocument has a known token stream field.
+ * Checks for "tokens" first (current KorAP-XML-Krill format),
+ * then falls back to "base" (legacy format).
+ * A document without either field will not contribute
+ * to token/sentence/paragraph statistics and cannot be searched.
+ *
+ * @param doc The FieldDocument to check
+ * @return true if the document has a known token field with term vectors
+ */
+ public boolean hasTokenField (FieldDocument doc) {
+ if (doc == null)
+ return false;
+ IndexableField field = doc.doc.getField("tokens");
+ if (field == null)
+ field = doc.doc.getField("base");
+ return field != null && field.fieldType().storeTermVectors();
+ }
+
+
+ /**
* Add a document to the index as a {@link FieldDocument}.
*
* @param doc
@@ -619,6 +648,16 @@
if (doc == null)
return doc;
+ if (!hasTokenField(doc)) {
+ log.error(
+ "Rejecting document '{}': no token stream - "
+ + "the document would not contribute to statistics "
+ + "and cannot be searched",
+ doc.getTextSigle()
+ );
+ return doc;
+ }
+
try {
// Add document to writer
diff --git a/src/main/java/de/ids_mannheim/korap/index/MetaExporter.java b/src/main/java/de/ids_mannheim/korap/index/MetaExporter.java
new file mode 100644
index 0000000..adcb5a3
--- /dev/null
+++ b/src/main/java/de/ids_mannheim/korap/index/MetaExporter.java
@@ -0,0 +1,357 @@
+package de.ids_mannheim.korap.index;
+
+import java.io.BufferedWriter;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintStream;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.TreeSet;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.DefaultParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.IndexableField;
+import org.apache.lucene.index.LeafReader;
+import org.apache.lucene.index.LeafReaderContext;
+import org.apache.lucene.store.MMapDirectory;
+import org.apache.lucene.util.Bits;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+/**
+ * Standalone tool to export the per-document metadata stored in an existing
+ * Krill (Lucene) index, without going through the web service API.
+ *
+ * <p>All metadata in a Krill index is kept as Lucene <i>stored fields</i>; only
+ * the primary-data fields {@code tokens} and {@code base} carry the heavy
+ * annotation streams and are skipped here. This tool iterates over every live
+ * (non-deleted) document and prints the requested fields as TSV, CSV or
+ * line-delimited JSON.</p>
+ *
+ * <p>Typical use: dump the {@code textSigle} column of two index instances and
+ * diff the sorted lists to find which texts are missing from / surplus in one
+ * of them.</p>
+ *
+ * <pre>
+ * Usage:
+ *
+ * # List which metadata fields exist in the index
+ * java -cp Krill.jar de.ids_mannheim.korap.index.MetaExporter \
+ * -i /path/to/index --list-fields
+ *
+ * # Dump just the text sigles (one per line)
+ * java -cp Krill.jar de.ids_mannheim.korap.index.MetaExporter \
+ * -i /path/to/index -f textSigle
+ *
+ * # Dump several columns as TSV with a header row
+ * java -cp Krill.jar de.ids_mannheim.korap.index.MetaExporter \
+ * -i /path/to/index -f textSigle,author,title,pubDate
+ *
+ * # Dump all stored metadata fields of every document as JSON lines
+ * java -cp Krill.jar de.ids_mannheim.korap.index.MetaExporter \
+ * -i /path/to/index --format json
+ * </pre>
+ *
+ * @author Krill
+ */
+public class MetaExporter {
+
+ // Primary-data fields that are never metadata and may be huge.
+ private static final Set<String> SKIP_FIELDS =
+ new LinkedHashSet<>(Arrays.asList("tokens", "base"));
+
+ private final DirectoryReader reader;
+
+ public MetaExporter (String indexPath) throws IOException {
+ this.reader = DirectoryReader.open(
+ new MMapDirectory(Paths.get(indexPath)));
+ }
+
+ public void close () throws IOException {
+ this.reader.close();
+ }
+
+ /**
+ * Collect the sorted union of all stored metadata field names across the
+ * whole index.
+ */
+ public Set<String> collectFieldNames () throws IOException {
+ Set<String> names = new TreeSet<>();
+ for (LeafReaderContext lrc : reader.leaves()) {
+ LeafReader lr = lrc.reader();
+ Bits liveDocs = lr.getLiveDocs();
+ int max = lr.maxDoc();
+ for (int i = 0; i < max; i++) {
+ if (liveDocs != null && !liveDocs.get(i))
+ continue;
+ Document doc = lr.document(i);
+ for (IndexableField f : doc.getFields()) {
+ String name = f.name();
+ if (!SKIP_FIELDS.contains(name))
+ names.add(name);
+ }
+ }
+ }
+ return names;
+ }
+
+ private enum Format {
+ TSV, CSV, JSON
+ }
+
+ /**
+ * Export the selected fields of every live document.
+ *
+ * @param fields
+ * ordered list of field names to output; if {@code null} the
+ * sorted union of all stored fields is used.
+ * @param format
+ * output format.
+ * @param header
+ * whether to emit a header row (TSV/CSV only).
+ * @param multiSep
+ * separator used to join multi-valued fields (TSV/CSV only).
+ */
+ public long export (List<String> fields, Format format, boolean header,
+ String multiSep, Writer out) throws IOException {
+
+ // Resolve field list lazily for tabular output if not given
+ if (fields == null && format != Format.JSON) {
+ fields = new ArrayList<>(collectFieldNames());
+ }
+
+ char sep = (format == Format.CSV) ? ',' : '\t';
+ ObjectMapper mapper = (format == Format.JSON) ? new ObjectMapper() : null;
+
+ if (format != Format.JSON && header) {
+ writeRow(out, fields, format, sep, null);
+ }
+
+ long count = 0;
+ for (LeafReaderContext lrc : reader.leaves()) {
+ LeafReader lr = lrc.reader();
+ Bits liveDocs = lr.getLiveDocs();
+ int max = lr.maxDoc();
+ for (int i = 0; i < max; i++) {
+ if (liveDocs != null && !liveDocs.get(i))
+ continue;
+ Document doc = lr.document(i);
+
+ if (format == Format.JSON) {
+ writeJson(out, mapper, doc, fields);
+ }
+ else {
+ List<String> values = new ArrayList<>(fields.size());
+ for (String name : fields) {
+ String[] vs = doc.getValues(name);
+ values.add(vs.length == 0 ? ""
+ : String.join(multiSep, vs));
+ }
+ writeRow(out, values, format, sep, null);
+ }
+ count++;
+ }
+ }
+ out.flush();
+ return count;
+ }
+
+ private void writeJson (Writer out, ObjectMapper mapper, Document doc,
+ List<String> fields) throws IOException {
+ ObjectNode node = mapper.createObjectNode();
+ if (fields != null) {
+ for (String name : fields) {
+ String[] vs = doc.getValues(name);
+ if (vs.length == 1)
+ node.put(name, vs[0]);
+ else if (vs.length > 1) {
+ for (String v : vs)
+ node.withArray(name).add(v);
+ }
+ }
+ }
+ else {
+ // All stored fields except the primary-data ones
+ for (IndexableField f : doc.getFields()) {
+ String name = f.name();
+ if (SKIP_FIELDS.contains(name))
+ continue;
+ String v = f.stringValue();
+ if (v == null)
+ continue;
+ if (node.has(name)) {
+ node.withArray(name).add(v);
+ }
+ else {
+ node.put(name, v);
+ }
+ }
+ }
+ out.write(mapper.writeValueAsString(node));
+ out.write('\n');
+ }
+
+ private void writeRow (Writer out, List<String> values, Format format,
+ char sep, String unused) throws IOException {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < values.size(); i++) {
+ if (i > 0)
+ sb.append(sep);
+ sb.append(escape(values.get(i), format, sep));
+ }
+ sb.append('\n');
+ out.write(sb.toString());
+ }
+
+ private String escape (String value, Format format, char sep) {
+ if (value == null)
+ value = "";
+ if (format == Format.CSV) {
+ boolean needQuote = value.indexOf(sep) >= 0
+ || value.indexOf('"') >= 0
+ || value.indexOf('\n') >= 0
+ || value.indexOf('\r') >= 0;
+ if (needQuote) {
+ return '"' + value.replace("\"", "\"\"") + '"';
+ }
+ return value;
+ }
+ // TSV: strip embedded tabs/newlines so the row stays intact
+ return value.replace('\t', ' ').replace('\n', ' ').replace('\r', ' ');
+ }
+
+ public static void main (String[] argv) {
+ Options options = new Options();
+ options.addOption(Option.builder("i").longOpt("index")
+ .desc("index directory to read (required)").hasArg()
+ .argName("index dir").required().build());
+ options.addOption(Option.builder("f").longOpt("fields")
+ .desc("comma-separated list of metadata fields/columns to "
+ + "export. Defaults to all stored fields.")
+ .hasArg().argName("field,field,...").build());
+ options.addOption(Option.builder("o").longOpt("output")
+ .desc("output file (defaults to stdout).").hasArg()
+ .argName("file").build());
+ options.addOption(Option.builder().longOpt("format")
+ .desc("output format: tsv (default), csv or json (one JSON "
+ + "object per line).")
+ .hasArg().argName("tsv|csv|json").build());
+ options.addOption(Option.builder().longOpt("no-header")
+ .desc("do not print a header row (tsv/csv).").build());
+ options.addOption(Option.builder().longOpt("multi-sep")
+ .desc("separator for multi-valued fields in tsv/csv "
+ + "(default '|').")
+ .hasArg().argName("sep").build());
+ options.addOption(Option.builder().longOpt("list-fields")
+ .desc("only list the metadata field names present in the "
+ + "index and exit.")
+ .build());
+
+ CommandLineParser parser = new DefaultParser();
+ CommandLine cmd;
+ try {
+ cmd = parser.parse(options, argv);
+ }
+ catch (ParseException e) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp(
+ "Krill metadata exporter\n java -cp Krill.jar "
+ + "de.ids_mannheim.korap.index.MetaExporter "
+ + "-i <index dir> [-f <fields>] "
+ + "[--format tsv|csv|json] [-o <file>] "
+ + "[--list-fields]",
+ options);
+ System.err.println("\n" + e.getMessage());
+ return;
+ }
+
+ String indexPath = cmd.getOptionValue("i");
+ MetaExporter exporter = null;
+ try {
+ exporter = new MetaExporter(indexPath);
+
+ if (cmd.hasOption("list-fields")) {
+ PrintStream ps = System.out;
+ for (String name : exporter.collectFieldNames()) {
+ ps.println(name);
+ }
+ return;
+ }
+
+ Format format = Format.TSV;
+ String fmt = cmd.getOptionValue("format");
+ if (fmt != null) {
+ switch (fmt.toLowerCase()) {
+ case "tsv": format = Format.TSV; break;
+ case "csv": format = Format.CSV; break;
+ case "json": format = Format.JSON; break;
+ default:
+ System.err.println("Unknown format: " + fmt);
+ return;
+ }
+ }
+
+ List<String> fields = null;
+ if (cmd.hasOption("f")) {
+ fields = new ArrayList<>();
+ for (String part : cmd.getOptionValue("f").split(",")) {
+ String t = part.trim();
+ if (!t.isEmpty())
+ fields.add(t);
+ }
+ }
+
+ boolean header = !cmd.hasOption("no-header");
+ String multiSep = cmd.getOptionValue("multi-sep", "|");
+
+ OutputStream os;
+ boolean closeOut = false;
+ if (cmd.hasOption("o")) {
+ os = Files.newOutputStream(Paths.get(cmd.getOptionValue("o")));
+ closeOut = true;
+ }
+ else {
+ os = System.out;
+ }
+
+ Writer out = new BufferedWriter(
+ new OutputStreamWriter(os, StandardCharsets.UTF_8));
+ long n = exporter.export(fields, format, header, multiSep, out);
+ out.flush();
+ if (closeOut)
+ out.close();
+
+ System.err.println("Exported " + n + " documents.");
+ }
+ catch (IOException e) {
+ System.err.println("Error: " + e.getMessage());
+ e.printStackTrace();
+ }
+ finally {
+ if (exporter != null) {
+ try {
+ exporter.close();
+ }
+ catch (IOException e) {
+ // ignore
+ }
+ }
+ }
+ }
+}
diff --git a/src/main/java/de/ids_mannheim/korap/query/spans/WithinSpans.java b/src/main/java/de/ids_mannheim/korap/query/spans/WithinSpans.java
index fe507d9..bfdfefa 100644
--- a/src/main/java/de/ids_mannheim/korap/query/spans/WithinSpans.java
+++ b/src/main/java/de/ids_mannheim/korap/query/spans/WithinSpans.java
@@ -204,17 +204,31 @@
log.trace("In the next embedded branch");
WithinSpan current = null;
+ WithinSpan candidate;
- // New - fetch until theres a span in the correct doc or bigger
+ // Skip stale entries, stop at future documents
while (!this.spanStore2.isEmpty()) {
- current = spanStore2.removeFirst();
- if (current.doc >= this.wrapDoc)
+ candidate = spanStore2.peekFirst();
+ if (candidate.doc < this.wrapDoc) {
+ spanStore2.removeFirst();
+ }
+ else if (candidate.doc == this.wrapDoc) {
+ current = spanStore2.removeFirst();
break;
- };
+ }
+ else {
+ break;
+ }
+ }
-
- // There is nothing in the second store
if (current == null) {
+
+ // Future-doc entries remain: advance wrap instead
+ if (!this.spanStore2.isEmpty()) {
+ this.nextSpanA();
+ continue;
+ }
+
if (DEBUG)
log.trace("SpanStore 2 is empty");
@@ -626,32 +640,49 @@
log.trace("skipTo document {}/{} -> {}", this.embeddedDoc,
this.wrapDoc, target);
- // Initialize spans
if (!this.init())
return false;
- assert target > this.embeddedDoc;
+ // Already at or past target: just find next match
+ if (this.matchDoc >= target)
+ return this.next() && this.matchDoc >= target;
- // Only forward embedded spans
- if (this.more && (this.embeddedDoc < target)) {
- if (this.embeddedSpans.skipTo(target)) {
- this.inSameDoc = false;
- this.embeddedStart = -1;
- this.embeddedEnd = -1;
- this.embeddedPayload = null;
- this.embeddedDoc = this.embeddedSpans.doc();
- }
+ // Fast-forward operands when both are behind target
+ if (this.embeddedDoc >= 0 && this.wrapDoc < target
+ && this.embeddedDoc < target) {
- // Can't be skipped to target
- else {
- this.inSameDoc = false;
- this.more = false;
+ if (!this.wrapSpans.skipTo(target))
return false;
- };
- };
+ this.wrapDoc = this.wrapSpans.doc();
+ if (this.wrapDoc == DocIdSetIterator.NO_MORE_DOCS)
+ return false;
- // Move to same doc
- return this.toSameDoc();
+ if (!this.embeddedSpans.skipTo(this.wrapDoc))
+ return false;
+ this.embeddedDoc = this.embeddedSpans.doc();
+ if (this.embeddedDoc == DocIdSetIterator.NO_MORE_DOCS)
+ return false;
+
+ // Reset internal state after skip
+ this.spanStore1.clear();
+ this.spanStore2.clear();
+ this.wrapStart = this.wrapSpans.start();
+ this.embeddedStart = this.embeddedSpans.start();
+ this.wrapEnd = -1;
+ this.embeddedEnd = -1;
+ this.embeddedPayload = null;
+ this.inSameDoc = (this.wrapDoc == this.embeddedDoc);
+ this.more = true;
+ this.tryMatch = true;
+ this.nextSpanB = true;
+ }
+
+ while (true) {
+ if (!this.next())
+ return false;
+ if (this.matchDoc >= target)
+ return true;
+ }
};
diff --git a/src/test/java/de/ids_mannheim/korap/TestMetaExporter.java b/src/test/java/de/ids_mannheim/korap/TestMetaExporter.java
new file mode 100644
index 0000000..cff3dbe
--- /dev/null
+++ b/src/test/java/de/ids_mannheim/korap/TestMetaExporter.java
@@ -0,0 +1,144 @@
+package de.ids_mannheim.korap;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.nio.file.Files;
+
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import de.ids_mannheim.korap.index.Indexer;
+import de.ids_mannheim.korap.index.MetaExporter;
+
+/**
+ * Tests for the standalone {@link MetaExporter} tool, which dumps the stored
+ * per-document metadata of an existing index.
+ *
+ * <p>An index containing a single known document (textSigle
+ * {@code BZK_D59.00089}) is built once from {@code src/test/resources/bzk}
+ * and reused by all tests.</p>
+ *
+ * @author Krill
+ */
+public class TestMetaExporter {
+
+ private static File tempBaseDirectory;
+ private static String indexDir;
+
+ private final ByteArrayOutputStream outContent = new ByteArrayOutputStream();
+ private final PrintStream originalOut = System.out;
+
+ @BeforeClass
+ public static void buildIndex () throws IOException {
+ tempBaseDirectory = Files.createTempDirectory("krill-export-test")
+ .toFile();
+ indexDir = new File(tempBaseDirectory, "index").getAbsolutePath();
+
+ // The Indexer prints to stdout; silence it during setup.
+ PrintStream original = System.out;
+ System.setOut(new PrintStream(new ByteArrayOutputStream()));
+ try {
+ Indexer.main(new String[] { "-c",
+ "src/test/resources/krill.properties", "-i",
+ "src/test/resources/bzk", "-o", indexDir });
+ }
+ finally {
+ System.setOut(original);
+ }
+ }
+
+ @Before
+ public void redirectOut () {
+ System.setOut(new PrintStream(outContent));
+ }
+
+ private void restoreOut () {
+ System.setOut(originalOut);
+ }
+
+ @Test
+ public void testListFields () {
+ MetaExporter.main(new String[] { "-i", indexDir, "--list-fields" });
+ restoreOut();
+ String out = outContent.toString();
+ // The text sigle is the key field used for diffing instances.
+ assertTrue("expected textSigle among listed fields",
+ out.contains("textSigle"));
+ assertTrue("expected corpusSigle among listed fields",
+ out.contains("corpusSigle"));
+ // The heavy primary-data fields must never be reported as metadata.
+ assertTrue("tokens must not be listed as a metadata field",
+ !out.contains("tokens\n"));
+ }
+
+ @Test
+ public void testExportSingleColumnNoHeader () {
+ MetaExporter.main(new String[] { "-i", indexDir, "-f", "textSigle",
+ "--no-header" });
+ restoreOut();
+ assertEquals("BZK_D59.00089", outContent.toString().trim());
+ }
+
+ @Test
+ public void testExportTsvHeader () {
+ MetaExporter.main(
+ new String[] { "-i", indexDir, "-f", "textSigle,corpusSigle" });
+ restoreOut();
+ String[] lines = outContent.toString().split("\n");
+ assertEquals("textSigle\tcorpusSigle", lines[0]);
+ assertEquals("BZK_D59.00089\tBZK", lines[1]);
+ }
+
+ @Test
+ public void testExportCsv () {
+ MetaExporter.main(new String[] { "-i", indexDir, "-f", "textSigle",
+ "--format", "csv", "--no-header" });
+ restoreOut();
+ assertEquals("BZK_D59.00089", outContent.toString().trim());
+ }
+
+ @Test
+ public void testExportJson () {
+ MetaExporter.main(
+ new String[] { "-i", indexDir, "--format", "json" });
+ restoreOut();
+ String line = outContent.toString().trim();
+ assertTrue("JSON line should be an object", line.startsWith("{"));
+ assertTrue("JSON should contain the textSigle",
+ line.contains("\"textSigle\":\"BZK_D59.00089\""));
+ // tokens/base must be excluded from the JSON dump as well.
+ assertTrue("JSON must not contain the tokens primary-data field",
+ !line.contains("\"tokens\":"));
+ }
+
+ @Test
+ public void testMissingIndexArgumentPrintsHelp () {
+ // Missing required -i must be handled gracefully (help printed, no throw).
+ MetaExporter.main(new String[] { "--list-fields" });
+ restoreOut();
+ assertTrue("expected usage/help output",
+ outContent.toString().contains("Krill metadata exporter")
+ || outContent.toString().isEmpty());
+ }
+
+ @AfterClass
+ public static void cleanup () {
+ if (tempBaseDirectory != null && tempBaseDirectory.exists())
+ deleteFile(tempBaseDirectory);
+ }
+
+ private static void deleteFile (File path) {
+ if (path.isDirectory() && path.list() != null) {
+ for (String filename : path.list())
+ deleteFile(new File(path, filename));
+ }
+ path.delete();
+ }
+}
diff --git a/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionIndex.java b/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionIndex.java
index 8b5fcef..eaaba0a 100644
--- a/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionIndex.java
@@ -1296,6 +1296,8 @@
for (int i = 0; i < 6000; i++) {
FieldDocument fd = new FieldDocument();
fd.addString("UID", Integer.toString(i));
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
if (i == 4500)
ki.commit();
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java b/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
index a7b3777..87ad55d 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
@@ -21,9 +21,11 @@
import static de.ids_mannheim.korap.TestSimple.*;
import de.ids_mannheim.korap.Krill;
+import de.ids_mannheim.korap.KrillCollection;
import de.ids_mannheim.korap.KrillIndex;
import de.ids_mannheim.korap.KrillMeta;
import de.ids_mannheim.korap.KrillQuery;
+import de.ids_mannheim.korap.collection.CollectionBuilder;
import de.ids_mannheim.korap.query.QueryBuilder;
import de.ids_mannheim.korap.query.wrap.SpanQueryWrapper;
import de.ids_mannheim.korap.response.Match;
@@ -637,6 +639,245 @@
};
+ /**
+ * A document with only metadata (no token stream / no "data" section)
+ * should be rejected by addDoc with an error.
+ */
+ @Test
+ public void testAddDocRejectsMetadataOnlyDocument () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/META/001");
+ fd.addString("author", "Test Author");
+ fd.addDate("pubDate", 20200101);
+
+ FieldDocument result = ki.addDoc(fd);
+ ki.commit();
+
+ assertEquals(0, ki.numberOf("documents"));
+ assertTrue(
+ "addDoc should return the doc even when rejected",
+ result != null
+ );
+ }
+
+ /**
+ * A document with only metadata should also be rejected via upsertDoc.
+ */
+ @Test
+ public void testUpsertDocRejectsMetadataOnlyDocument () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/META/002");
+ fd.addString("author", "Test Author");
+
+ FieldDocument result = ki.upsertDoc(fd);
+ ki.commit();
+
+ assertEquals(0, ki.numberOf("documents"));
+ }
+
+ /**
+ * A JSON document with fields but no data section should be rejected.
+ */
+ @Test
+ public void testAddDocRejectsJsonWithoutTokenStream () throws Exception {
+ String json = "{"
+ + " \"fields\" : ["
+ + " {"
+ + " \"@type\" : \"koral:field\","
+ + " \"type\" : \"type:string\","
+ + " \"key\" : \"textSigle\","
+ + " \"value\" : \"TEST/NODATA/001\""
+ + " },"
+ + " {"
+ + " \"@type\" : \"koral:field\","
+ + " \"type\" : \"type:text\","
+ + " \"key\" : \"author\","
+ + " \"value\" : \"Nobody\""
+ + " },"
+ + " {"
+ + " \"@type\" : \"koral:field\","
+ + " \"type\" : \"type:date\","
+ + " \"key\" : \"pubDate\","
+ + " \"value\" : \"2020-01-01\""
+ + " }"
+ + " ]"
+ + "}";
+
+ KrillIndex ki = new KrillIndex();
+ FieldDocument fd = ki.addDoc(json);
+ ki.commit();
+
+ assertEquals(
+ "Metadata-only document should not be indexed",
+ 0, ki.numberOf("documents")
+ );
+ }
+
+ /**
+ * A document with a token stream on an unknown field name
+ * (neither "tokens" nor "base") should be rejected.
+ */
+ @Test
+ public void testAddDocRejectsUnknownTokenFieldName () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/WRONG/001");
+ fd.addTV("other", "hello world",
+ "[(0-5)s:hello|i:hello|_0$<i>0<i>5|-:tokens$<i>2]"
+ + "[(6-11)s:world|i:world|_1$<i>6<i>11]");
+ ki.addDoc(fd);
+ ki.commit();
+
+ assertEquals(
+ "Document with token stream on unknown field should be rejected",
+ 0, ki.numberOf("documents")
+ );
+ }
+
+ /**
+ * A document with a token stream on the legacy "base" field
+ * should be accepted.
+ */
+ @Test
+ public void testAddDocAcceptsLegacyBaseField () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/BASE/001");
+ fd.addTV("base", "hello world",
+ "[(0-5)s:hello|i:hello|_0$<i>0<i>5|-:t$<i>2]"
+ + "[(6-11)s:world|i:world|_1$<i>6<i>11]");
+ ki.addDoc(fd);
+ ki.commit();
+
+ assertEquals(
+ "Document with legacy 'base' field should be accepted",
+ 1, ki.numberOf("documents")
+ );
+ }
+
+ /**
+ * When metadata-only documents are mixed with proper documents,
+ * only proper documents should be indexed and contribute to stats.
+ */
+ @Test
+ public void testStatsWithMixedValidAndMetadataOnlyDocuments () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ // Valid document with token stream
+ FieldDocument fd1 = new FieldDocument();
+ fd1.addString("textSigle", "TEST/OK/001");
+ fd1.addString("author", "Good Author");
+ fd1.addTV("tokens", "hello world",
+ "[(0-5)s:hello|i:hello|_0$<i>0<i>5|-:tokens$<i>2]"
+ + "[(6-11)s:world|i:world|_1$<i>6<i>11]");
+ ki.addDoc(fd1);
+
+ // Metadata-only document (should be rejected)
+ FieldDocument fd2 = new FieldDocument();
+ fd2.addString("textSigle", "TEST/BAD/001");
+ fd2.addString("author", "Bad Author");
+ ki.addDoc(fd2);
+
+ // Another valid document
+ FieldDocument fd3 = new FieldDocument();
+ fd3.addString("textSigle", "TEST/OK/002");
+ fd3.addString("author", "Another Author");
+ fd3.addTV("tokens", "foo bar baz",
+ "[(0-3)s:foo|i:foo|_0$<i>0<i>3|-:tokens$<i>3]"
+ + "[(4-7)s:bar|i:bar|_1$<i>4<i>7]"
+ + "[(8-11)s:baz|i:baz|_2$<i>8<i>11]");
+ ki.addDoc(fd3);
+
+ ki.commit();
+
+ assertEquals(
+ "Only valid documents should be counted",
+ 2, ki.numberOf("documents")
+ );
+ assertEquals(5, ki.numberOf("tokens"));
+ }
+
+ /**
+ * A document with an empty stream (data section present but stream
+ * is empty) has a token stream field but no tokens. This is allowed
+ * through indexing since the document structure is valid (it has a
+ * "data" section), but it contributes 0 to token statistics.
+ */
+ @Test
+ public void testAddDocAcceptsEmptyTokenStream () throws Exception {
+ String json = "{"
+ + " \"data\" : {"
+ + " \"text\" : \"\","
+ + " \"name\" : \"tokens\","
+ + " \"stream\" : []"
+ + " },"
+ + " \"fields\" : ["
+ + " {"
+ + " \"@type\" : \"koral:field\","
+ + " \"type\" : \"type:string\","
+ + " \"key\" : \"textSigle\","
+ + " \"value\" : \"TEST/EMPTY/001\""
+ + " }"
+ + " ]"
+ + "}";
+
+ KrillIndex ki = new KrillIndex();
+ FieldDocument fd = ki.addDoc(json);
+ ki.commit();
+
+ assertEquals(
+ "Document with empty stream is accepted (has token field)",
+ 1, ki.numberOf("documents")
+ );
+ assertEquals(
+ "Empty stream contributes 0 tokens",
+ 0, ki.numberOf("tokens")
+ );
+ }
+
+ /**
+ * Upserting a valid document with a metadata-only replacement should
+ * NOT remove the original and should reject the new one.
+ */
+ @Test
+ public void testUpsertDoesNotReplaceValidDocWithMetadataOnly () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ // First: add a valid document
+ FieldDocument fd1 = new FieldDocument();
+ fd1.addString("textSigle", "TEST/UPSERT/001");
+ fd1.addString("author", "Original");
+ fd1.addTV("tokens", "good data",
+ "[(0-4)s:good|i:good|_0$<i>0<i>4|-:tokens$<i>2]"
+ + "[(5-9)s:data|i:data|_1$<i>5<i>9]");
+ ki.addDoc(fd1);
+ ki.commit();
+
+ assertEquals(1, ki.numberOf("documents"));
+ assertEquals(2, ki.numberOf("tokens"));
+
+ // Now try to upsert with metadata-only (should be rejected)
+ FieldDocument fd2 = new FieldDocument();
+ fd2.addString("textSigle", "TEST/UPSERT/001");
+ fd2.addString("author", "Replacement Without Tokens");
+ ki.upsertDoc(fd2);
+ ki.commit();
+
+ // The original document should still be there since the upsert
+ // was rejected before the delete could happen
+ assertEquals(1, ki.numberOf("documents"));
+ assertEquals(2, ki.numberOf("tokens"));
+
+ MetaFields mfs = ki.getFields("TEST/UPSERT/001");
+ assertEquals("Original", mfs.getFieldValue("author"));
+ }
+
@Test
public void indexUpsert () throws Exception {
KrillIndex ki = new KrillIndex();
@@ -645,6 +886,8 @@
FieldDocument fd = new FieldDocument();
fd.addString("textSigle", "AAA/BBB/001");
fd.addString("content", "Example1");
+ fd.addTV("tokens", "Example1",
+ "[(0-8)s:Example1|i:example1|_0$<i>0<i>8|-:tokens$<i>1]");
ki.upsertDoc(fd);
ki.commit();
@@ -662,6 +905,8 @@
fd = new FieldDocument();
fd.addString("textSigle", "AAA/BBB/002");
fd.addString("content", "Example2");
+ fd.addTV("tokens", "Example2",
+ "[(0-8)s:Example2|i:example2|_0$<i>0<i>8|-:tokens$<i>1]");
ki.upsertDoc(fd);
ki.commit();
@@ -675,6 +920,8 @@
fd = new FieldDocument();
fd.addString("textSigle", "AAA/BBB/001");
fd.addString("content", "Example3");
+ fd.addTV("tokens", "Example3",
+ "[(0-8)s:Example3|i:example3|_0$<i>0<i>8|-:tokens$<i>1]");
ki.upsertDoc(fd);
ki.commit();
@@ -696,6 +943,8 @@
fd = new FieldDocument();
fd.addString("textSigle", "AAA/DDD/005");
fd.addString("content", "Example4");
+ fd.addTV("tokens", "Example4",
+ "[(0-8)s:Example4|i:example4|_0$<i>0<i>8|-:tokens$<i>1]");
ki.upsertDoc(fd);
ki.commit();
@@ -704,7 +953,873 @@
};
-
+
+ private static FieldDocument createSimpleDoc (String textSigle,
+ String author, int tokenCount) {
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", textSigle);
+ fd.addString("author", author);
+
+ StringBuilder primaryData = new StringBuilder();
+ StringBuilder tvBuilder = new StringBuilder();
+ for (int i = 0; i < tokenCount; i++) {
+ char c = (char) ('a' + (i % 26));
+ if (i > 0) primaryData.append(' ');
+ primaryData.append(c);
+ int start = i * 2;
+ int end = start + 1;
+ tvBuilder.append("[(").append(start).append("-").append(end)
+ .append(")s:").append(c).append("|i:").append(c)
+ .append("|_").append(i).append("$<i>").append(start)
+ .append("<i>").append(end);
+ if (i == 0) {
+ tvBuilder.append("|-:tokens$<i>").append(tokenCount);
+ }
+ tvBuilder.append("]");
+ }
+ fd.addTV("tokens", primaryData.toString(), tvBuilder.toString());
+ return fd;
+ }
+
+
+ @Test
+ public void testStatsAfterDeleteSingleSegment () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ assertEquals(3, ki.numberOf("documents"));
+ assertEquals(22, ki.numberOf("tokens"));
+
+ ki.delDocs("textSigle", "A/B/002");
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(12, ki.numberOf("tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsAfterDeleteMultipleSegments () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ assertEquals(3, ki.numberOf("documents"));
+ assertEquals(22, ki.numberOf("tokens"));
+
+ ki.delDocs("textSigle", "A/B/002");
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(12, ki.numberOf("tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsAfterUpsertSingleSegment () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(15, ki.numberOf("tokens"));
+
+ FieldDocument updated = createSimpleDoc("A/B/001", "Frank", 3);
+ ki.upsertDoc(updated);
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(13, ki.numberOf("tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsAfterUpsertMultipleSegments () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ assertEquals(3, ki.numberOf("documents"));
+ assertEquals(22, ki.numberOf("tokens"));
+
+ FieldDocument updated = createSimpleDoc("A/B/002", "Peter", 4);
+ ki.upsertDoc(updated);
+ ki.commit();
+
+ assertEquals(3, ki.numberOf("documents"));
+ assertEquals(16, ki.numberOf("tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithVCAfterDelete () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+
+ assertEquals(2, kc.docCount());
+ assertEquals(12, kc.numberOf("tokens", "tokens"));
+
+ ki.delDocs("textSigle", "A/B/001");
+ ki.commit();
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+
+ assertEquals(1, kc.docCount());
+ assertEquals(7, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithVCAfterDeleteMultipleSegments ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+
+ assertEquals(2, kc.docCount());
+ assertEquals(12, kc.numberOf("tokens", "tokens"));
+
+ ki.delDocs("textSigle", "A/B/001");
+ ki.commit();
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+
+ assertEquals(1, kc.docCount());
+ assertEquals(7, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithVCAfterUpsert () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+
+ assertEquals(2, kc.docCount());
+ assertEquals(12, kc.numberOf("tokens", "tokens"));
+
+ FieldDocument updated = createSimpleDoc("A/B/001", "Frank", 3);
+ ki.upsertDoc(updated);
+ ki.commit();
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+
+ assertEquals(2, kc.docCount());
+ assertEquals(10, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithVCAfterUpsertMultipleSegments ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+
+ assertEquals(2, kc.docCount());
+ assertEquals(12, kc.numberOf("tokens", "tokens"));
+
+ FieldDocument updated = createSimpleDoc("A/B/002", "Peter", 4);
+ ki.upsertDoc(updated);
+ ki.commit();
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Peter"));
+
+ assertEquals(1, kc.docCount());
+ assertEquals(4, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsConsistencyAfterMultipleUpserts ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(15, ki.numberOf("tokens"));
+
+ ki.upsertDoc(createSimpleDoc("A/B/001", "Frank", 3));
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(13, ki.numberOf("tokens"));
+
+ ki.upsertDoc(createSimpleDoc("A/B/001", "Frank", 8));
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(18, ki.numberOf("tokens"));
+
+ ki.upsertDoc(createSimpleDoc("A/B/002", "Peter", 2));
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(10, ki.numberOf("tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsAfterDeleteAllInSegment () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(15, ki.numberOf("tokens"));
+
+ ki.delDocs("textSigle", "A/B/001");
+ ki.commit();
+
+ assertEquals(1, ki.numberOf("documents"));
+ assertEquals(10, ki.numberOf("tokens"));
+
+ ki.delDocs("textSigle", "A/B/002");
+ ki.commit();
+
+ assertEquals(0, ki.numberOf("documents"));
+ assertEquals(0, ki.numberOf("tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsDocCountAndTokensConsistentWithVC ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ ki.upsertDoc(createSimpleDoc("A/B/001", "Frank", 3));
+ ki.commit();
+
+ ki.delDocs("textSigle", "A/B/002");
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(10, ki.numberOf("tokens"));
+
+ CollectionBuilder cb = new CollectionBuilder();
+
+ KrillCollection kcFrank = new KrillCollection(ki);
+ kcFrank.fromBuilder(cb.term("author", "Frank"));
+ assertEquals(2, kcFrank.docCount());
+ assertEquals(10, kcFrank.numberOf("tokens", "tokens"));
+
+ KrillCollection kcPeter = new KrillCollection(ki);
+ kcPeter.fromBuilder(cb.term("author", "Peter"));
+ assertEquals(0, kcPeter.docCount());
+ assertEquals(0, kcPeter.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithNegatedVCAfterDelete () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ ki.delDocs("textSigle", "A/B/002");
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Peter").not());
+
+ assertEquals(2, kc.docCount());
+ assertEquals(12, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsAfterUpsertChangingAuthor () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ ki.upsertDoc(createSimpleDoc("A/B/001", "Peter", 5));
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+
+ KrillCollection kcFrank = new KrillCollection(ki);
+ kcFrank.fromBuilder(cb.term("author", "Frank"));
+ assertEquals(0, kcFrank.docCount());
+ assertEquals(0, kcFrank.numberOf("tokens", "tokens"));
+
+ KrillCollection kcPeter = new KrillCollection(ki);
+ kcPeter.fromBuilder(cb.term("author", "Peter"));
+ assertEquals(2, kcPeter.docCount());
+ assertEquals(15, kcPeter.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsAfterManyUpsertsAcrossSegments () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ for (int i = 1; i <= 10; i++) {
+ ki.addDoc(createSimpleDoc(
+ "C/D/" + String.format("%03d", i),
+ i <= 5 ? "Frank" : "Peter",
+ i * 2));
+ ki.commit();
+ }
+
+ assertEquals(10, ki.numberOf("documents"));
+ assertEquals(110, ki.numberOf("tokens"));
+
+ for (int i = 1; i <= 10; i++) {
+ ki.upsertDoc(createSimpleDoc(
+ "C/D/" + String.format("%03d", i),
+ i <= 5 ? "Frank" : "Peter",
+ i));
+ ki.commit();
+ }
+
+ assertEquals(10, ki.numberOf("documents"));
+ assertEquals(55, ki.numberOf("tokens"));
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kcFrank = new KrillCollection(ki);
+ kcFrank.fromBuilder(cb.term("author", "Frank"));
+ assertEquals(5, kcFrank.docCount());
+ assertEquals(15, kcFrank.numberOf("tokens", "tokens"));
+
+ KrillCollection kcPeter = new KrillCollection(ki);
+ kcPeter.fromBuilder(cb.term("author", "Peter"));
+ assertEquals(5, kcPeter.docCount());
+ assertEquals(40, kcPeter.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsDocCountVsNumberOfDocumentsAfterUpsert ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ ki.upsertDoc(createSimpleDoc("A/B/001", "Frank", 3));
+ ki.commit();
+
+ ki.upsertDoc(createSimpleDoc("A/B/002", "Peter", 8));
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+
+ long docCountMethod = kc.docCount();
+ long numberOfDocuments = kc.numberOf("documents");
+ assertEquals(docCountMethod, numberOfDocuments);
+ assertEquals(2, docCountMethod);
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(
+ cb.orGroup()
+ .with(cb.term("author", "Frank"))
+ .with(cb.term("author", "Peter")));
+ long allDocs = kc.docCount();
+ long allDocsNumberOf = kc.numberOf("documents");
+ assertEquals(allDocs, allDocsNumberOf);
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithDateFilterAfterUpsert () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd1 = createSimpleDoc("A/B/001", "Frank", 5);
+ fd1.addDate("pubDate", 20180315);
+ ki.addDoc(fd1);
+
+ FieldDocument fd2 = createSimpleDoc("A/B/002", "Peter", 10);
+ fd2.addDate("pubDate", 20180620);
+ ki.addDoc(fd2);
+
+ FieldDocument fd3 = createSimpleDoc("A/B/003", "Frank", 7);
+ fd3.addDate("pubDate", 20190101);
+ ki.addDoc(fd3);
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.date("pubDate", "2018"));
+
+ assertEquals(2, kc.docCount());
+ assertEquals(15, kc.numberOf("tokens", "tokens"));
+
+ FieldDocument updated = createSimpleDoc("A/B/001", "Frank", 3);
+ updated.addDate("pubDate", 20180315);
+ ki.upsertDoc(updated);
+ ki.commit();
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.date("pubDate", "2018"));
+
+ assertEquals(2, kc.docCount());
+ assertEquals(13, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithDateFilterAndMultipleSegmentsAfterUpsert ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd1 = createSimpleDoc("A/B/001", "Frank", 5);
+ fd1.addDate("pubDate", 20180315);
+ ki.addDoc(fd1);
+ ki.commit();
+
+ FieldDocument fd2 = createSimpleDoc("A/B/002", "Peter", 10);
+ fd2.addDate("pubDate", 20180620);
+ ki.addDoc(fd2);
+ ki.commit();
+
+ FieldDocument fd3 = createSimpleDoc("A/B/003", "Frank", 7);
+ fd3.addDate("pubDate", 20190101);
+ ki.addDoc(fd3);
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.date("pubDate", "2018"));
+ assertEquals(2, kc.docCount());
+ assertEquals(15, kc.numberOf("tokens", "tokens"));
+
+ FieldDocument updated = createSimpleDoc("A/B/001", "Frank", 3);
+ updated.addDate("pubDate", 20180315);
+ ki.upsertDoc(updated);
+ ki.commit();
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.date("pubDate", "2018"));
+ assertEquals(2, kc.docCount());
+ assertEquals(13, kc.numberOf("tokens", "tokens"));
+
+ FieldDocument updated2 = createSimpleDoc("A/B/002", "Peter", 4);
+ updated2.addDate("pubDate", 20180620);
+ ki.upsertDoc(updated2);
+ ki.commit();
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.date("pubDate", "2018"));
+ assertEquals(2, kc.docCount());
+ assertEquals(7, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithComplexVCAfterMultipleUpserts ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ for (int i = 1; i <= 5; i++) {
+ FieldDocument fd = createSimpleDoc(
+ "X/Y/" + String.format("%03d", i),
+ i <= 3 ? "Frank" : "Peter",
+ i * 3);
+ fd.addDate("pubDate", 20180100 + i);
+ ki.addDoc(fd);
+ ki.commit();
+ }
+
+ assertEquals(5, ki.numberOf("documents"));
+ assertEquals(45, ki.numberOf("tokens"));
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(
+ cb.andGroup()
+ .with(cb.date("pubDate", "2018"))
+ .with(cb.term("author", "Frank")));
+ assertEquals(3, kc.docCount());
+ assertEquals(18, kc.numberOf("tokens", "tokens"));
+
+ for (int i = 1; i <= 5; i++) {
+ FieldDocument fd = createSimpleDoc(
+ "X/Y/" + String.format("%03d", i),
+ i <= 3 ? "Frank" : "Peter",
+ i);
+ fd.addDate("pubDate", 20180100 + i);
+ ki.upsertDoc(fd);
+ ki.commit();
+ }
+
+ assertEquals(5, ki.numberOf("documents"));
+ assertEquals(15, ki.numberOf("tokens"));
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(
+ cb.andGroup()
+ .with(cb.date("pubDate", "2018"))
+ .with(cb.term("author", "Frank")));
+ assertEquals(3, kc.docCount());
+ assertEquals(6, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsWithNegationAndDeletedDocs ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ ki.addDoc(createSimpleDoc("A/B/004", "Maria", 3));
+ ki.commit();
+
+ ki.delDocs("textSigle", "A/B/002");
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank").not());
+
+ assertEquals(1, kc.docCount());
+ assertEquals(3, kc.numberOf("tokens", "tokens"));
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Peter").not());
+
+ assertEquals(3, kc.docCount());
+ assertEquals(15, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsUpsertThenBatchDelete () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ for (int i = 1; i <= 5; i++) {
+ ki.addDoc(createSimpleDoc(
+ "A/B/" + String.format("%03d", i), "Frank", i * 2));
+ ki.commit();
+ }
+
+ assertEquals(5, ki.numberOf("documents"));
+ assertEquals(30, ki.numberOf("tokens"));
+
+ ki.upsertDoc(createSimpleDoc("A/B/003", "Frank", 1));
+ ki.commit();
+
+ assertEquals(5, ki.numberOf("documents"));
+ assertEquals(25, ki.numberOf("tokens"));
+
+ ki.delDocs("textSigle", "A/B/001");
+ ki.delDocs("textSigle", "A/B/002");
+ ki.delDocs("textSigle", "A/B/004");
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(11, ki.numberOf("tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsVCMatchesOnlyDeletedDocs () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ ki.delDocs("textSigle", "A/B/002");
+ ki.commit();
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Peter"));
+ assertEquals(0, kc.docCount());
+ assertEquals(0, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsAfterCloseAndReopen () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(15, ki.numberOf("tokens"));
+
+ ki.close();
+
+ ki.upsertDoc(createSimpleDoc("A/B/001", "Frank", 3));
+ ki.commit();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(13, ki.numberOf("tokens"));
+
+ ki.close();
+
+ assertEquals(2, ki.numberOf("documents"));
+ assertEquals(13, ki.numberOf("tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsMultipleUpsertsWithoutIntermediateCommit ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ assertEquals(3, ki.numberOf("documents"));
+ assertEquals(22, ki.numberOf("tokens"));
+
+ ki.upsertDoc(createSimpleDoc("A/B/001", "Frank", 3));
+ ki.upsertDoc(createSimpleDoc("A/B/002", "Peter", 4));
+ ki.upsertDoc(createSimpleDoc("A/B/003", "Frank", 2));
+ ki.commit();
+
+ assertEquals(3, ki.numberOf("documents"));
+ assertEquals(9, ki.numberOf("tokens"));
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+ assertEquals(2, kc.docCount());
+ assertEquals(5, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsAfterForceMerge () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ ki.addDoc(createSimpleDoc("A/B/001", "Frank", 5));
+ ki.commit();
+ ki.addDoc(createSimpleDoc("A/B/002", "Peter", 10));
+ ki.commit();
+ ki.addDoc(createSimpleDoc("A/B/003", "Frank", 7));
+ ki.commit();
+
+ ki.upsertDoc(createSimpleDoc("A/B/001", "Frank", 3));
+ ki.commit();
+ ki.upsertDoc(createSimpleDoc("A/B/002", "Peter", 4));
+ ki.commit();
+
+ ki.writer().forceMerge(1);
+ ki.commit();
+
+ assertEquals(3, ki.numberOf("documents"));
+ assertEquals(14, ki.numberOf("tokens"));
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+ assertEquals(2, kc.docCount());
+ assertEquals(10, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
+
+ @Test
+ public void testStatsStressWithManySegments () throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ int numDocs = 100;
+ long expectedTotalTokens = 0;
+
+ for (int i = 1; i <= numDocs; i++) {
+ String author = (i % 3 == 0) ? "Peter" : "Frank";
+ int tokenCount = (i % 7) + 1;
+ FieldDocument fd = createSimpleDoc(
+ "S/T/" + String.format("%04d", i), author, tokenCount);
+ fd.addDate("pubDate", 20180100 + (i % 28) + 1);
+ ki.addDoc(fd);
+ if (i % 10 == 0) ki.commit();
+ expectedTotalTokens += tokenCount;
+ }
+ ki.commit();
+
+ assertEquals(numDocs, ki.numberOf("documents"));
+ assertEquals(expectedTotalTokens, ki.numberOf("tokens"));
+
+ long newTotalTokens = 0;
+ int newFrankDocs = 0;
+ long newFrankTokens = 0;
+
+ for (int i = 1; i <= numDocs; i++) {
+ String author = (i % 3 == 0) ? "Peter" : "Frank";
+ int tokenCount = (i % 5) + 1;
+ FieldDocument fd = createSimpleDoc(
+ "S/T/" + String.format("%04d", i), author, tokenCount);
+ fd.addDate("pubDate", 20180100 + (i % 28) + 1);
+ ki.upsertDoc(fd);
+ if (i % 10 == 0) ki.commit();
+ newTotalTokens += tokenCount;
+ if (author.equals("Frank")) {
+ newFrankDocs++;
+ newFrankTokens += tokenCount;
+ }
+ }
+ ki.commit();
+
+ assertEquals(numDocs, ki.numberOf("documents"));
+ assertEquals(newTotalTokens, ki.numberOf("tokens"));
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("author", "Frank"));
+ assertEquals(newFrankDocs, kc.docCount());
+ assertEquals(newFrankTokens, kc.numberOf("tokens", "tokens"));
+
+ kc = new KrillCollection(ki);
+ kc.fromBuilder(
+ cb.andGroup()
+ .with(cb.date("pubDate", "2018"))
+ .with(cb.term("author", "Frank")));
+ assertEquals(newFrankDocs, kc.docCount());
+ assertEquals(newFrankTokens, kc.numberOf("tokens", "tokens"));
+
+ ki.close();
+ }
+
private static String createDocString1 () {
return new String(
"{"
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestIndexRevision.java b/src/test/java/de/ids_mannheim/korap/index/TestIndexRevision.java
index 1c43baf..d6d8278 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestIndexRevision.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestIndexRevision.java
@@ -3,6 +3,7 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.assertFalse;
import java.util.List;
@@ -209,4 +210,50 @@
assertEquals("Wes8Bd4h1OypPqbWF5njeQ==",ki.getFingerprint());
};
+
+ @Test
+ public void testCommitForceFalseDoesNotCloseReaderWhenNoChanges ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/DOC/001");
+ fd.addString("content", "Example1");
+ ki.addDoc(fd);
+ ki.commit();
+
+ // Ensure reader is open (and there are no pending changes)
+ ki.reader();
+ assertTrue(ki.isReaderOpen());
+
+ // With no pending changes, force=false must NOT trigger a commit
+ // (and therefore must not close the reader)
+ ki.commit(false);
+ assertTrue(ki.isReaderOpen());
+
+ ki.close();
+ }
+
+ @Test
+ public void testCommitForceTrueClosesReaderWhenNoChanges ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/DOC/001");
+ fd.addString("content", "Example1");
+ ki.addDoc(fd);
+ ki.commit();
+
+ // Ensure reader is open (and there are no pending changes)
+ ki.reader();
+ assertTrue(ki.isReaderOpen());
+
+ // With no pending changes, force=true must still trigger a commit,
+ // which closes the reader
+ ki.commit(true);
+ assertFalse(ki.isReaderOpen());
+
+ ki.close();
+ };
};
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
index e6c7d93..a301544 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
@@ -114,12 +114,16 @@
FieldDocument fd = new FieldDocument();
fd.addString("name", "Peter");
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
assertEquals(0, ki.numberOf("base", "documents"));
fd = new FieldDocument();
fd.addString("name", "Michael");
+ fd.addTV("tokens", "y",
+ "[(0-1)s:y|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
assertEquals(0, ki.numberOf("base", "documents"));
@@ -170,11 +174,15 @@
FieldDocument fd = new FieldDocument();
fd.addText("title", "Peter");
fd.setUID(22);
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
fd = new FieldDocument();
fd.addText("title", "Akron");
fd.setUID("05678");
+ fd.addTV("tokens", "y",
+ "[(0-1)s:y|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
ki.commit();
@@ -211,6 +219,8 @@
fd.addText("title", "Der Name der Rose");
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
/* Save documents */
@@ -335,6 +345,8 @@
FieldDocument fd = new FieldDocument();
fd.addString("name", "Peter");
fd.addString("textSigle", "a/b/c");
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.upsertDoc(fd);
/* Save documents */
@@ -343,6 +355,8 @@
fd = new FieldDocument();
fd.addString("name", "Frank");
fd.addString("textSigle", "a/b/d");
+ fd.addTV("tokens", "y",
+ "[(0-1)s:y|_0$<i>0<i>1|-:tokens$<i>1]");
ki.upsertDoc(fd);
/* Save documents */
@@ -351,6 +365,8 @@
fd = new FieldDocument();
fd.addString("name", "Franz");
fd.addString("textSigle", "a/b/c");
+ fd.addTV("tokens", "z",
+ "[(0-1)s:z|_0$<i>0<i>1|-:tokens$<i>1]");
ki.upsertDoc(fd);
/* Save documents */
@@ -384,11 +400,15 @@
FieldDocument fd = new FieldDocument();
fd.addString("textSigle", "aaaa");
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
fd = new FieldDocument();
fd.addString("textSigle", "bbbb");
fd.setUID("05678");
+ fd.addTV("tokens", "y",
+ "[(0-1)s:y|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
ki.commit();
@@ -413,6 +433,8 @@
fd = new FieldDocument();
fd.addString("textSigle", "cccc");
+ fd.addTV("tokens", "z",
+ "[(0-1)s:z|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
ki.commit();
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestPrimaryDataProtection.java b/src/test/java/de/ids_mannheim/korap/index/TestPrimaryDataProtection.java
index 11a6421..191ced2 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestPrimaryDataProtection.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestPrimaryDataProtection.java
@@ -385,6 +385,10 @@
fd.addString("textSigle", "TST-004-0001");
fd.addText("title", "Custom Field Document");
fd.setUID(45);
+ fd.addTV("tokens", "some indexed text",
+ "[(0-4)s:some|_0#0-4|-:t$<i>3]"
+ + "[(5-12)s:indexed|_1#5-12]"
+ + "[(13-17)s:text|_2#13-17]");
fd.addTV("customTokens", "leaked custom text",
"[(0-6)s:leaked|_0#0-6|-:t$<i>3]"
+ "[(7-13)s:custom|_1#7-13]"
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestWithinIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestWithinIndex.java
index b7c2416..0f18b1b 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestWithinIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestWithinIndex.java
@@ -3,27 +3,46 @@
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import static org.junit.Assume.assumeTrue;
import java.io.*;
import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+import java.util.TreeSet;
+
import static de.ids_mannheim.korap.TestSimple.*;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermContext;
+import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.search.spans.SpanQuery;
+import org.apache.lucene.search.spans.Spans;
import org.apache.lucene.search.spans.SpanTermQuery;
+import org.apache.lucene.util.Bits;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import de.ids_mannheim.korap.Krill;
+import de.ids_mannheim.korap.KrillCollection;
import de.ids_mannheim.korap.KrillIndex;
import de.ids_mannheim.korap.KrillQuery;
+import de.ids_mannheim.korap.collection.CollectionBuilder;
import de.ids_mannheim.korap.query.QueryBuilder;
import de.ids_mannheim.korap.query.SpanClassQuery;
import de.ids_mannheim.korap.query.SpanElementQuery;
+import de.ids_mannheim.korap.query.SpanFocusQuery;
import de.ids_mannheim.korap.query.wrap.SpanQueryWrapper;
import de.ids_mannheim.korap.query.SpanNextQuery;
import de.ids_mannheim.korap.query.SpanWithinQuery;
import de.ids_mannheim.korap.query.wrap.SpanQueryWrapper;
+import de.ids_mannheim.korap.response.Match;
import de.ids_mannheim.korap.response.Result;
import de.ids_mannheim.korap.util.QueryException;
@@ -1169,4 +1188,890 @@
Result kr = ki.search(sq, (short) 1);
assertEquals(1, kr.getTotalResults());
}
+
+
+ // Build one "tokens" FieldDocument from sentence token arrays.
+ // Each input array is one sentence, e.g. ["b","a","w2"].
+ // Example doc: [["b","a","w2"],["a","w4","w5"]].
+ private FieldDocument multiSentenceDoc (String[]... sentences) {
+ FieldDocument fd = new FieldDocument();
+
+ int total = 0;
+ for (String[] s : sentences) total += s.length;
+
+ StringBuilder surface = new StringBuilder();
+ StringBuilder tv = new StringBuilder();
+
+ int tokIdx = 0;
+ int charOff = 0;
+ int sentStart = 0;
+
+ for (int si = 0; si < sentences.length; si++) {
+ String[] sent = sentences[si];
+ int sentEnd = sentStart + sent.length;
+
+ for (int ti = 0; ti < sent.length; ti++) {
+ String tok = sent[ti];
+ int cStart = charOff;
+ int cEnd = charOff + tok.length();
+
+ surface.append(tok);
+
+ tv.append("[(").append(cStart).append("-").append(cEnd)
+ .append(")");
+ tv.append("s:").append(tok);
+
+ if (tokIdx == 0) {
+ int totalChars = 0;
+ for (String[] s : sentences)
+ for (String t : s)
+ totalChars += t.length();
+ tv.append("|-:t$<i>").append(total);
+ tv.append("|<>:base/s:t$<b>64<i>0<i>")
+ .append(totalChars).append("<i>")
+ .append(total).append("<b>0");
+ }
+ if (ti == 0) {
+ int sentCharEnd = cStart;
+ for (String t : sent) sentCharEnd += t.length();
+ tv.append("|<>:base/s:s$<b>64<i>")
+ .append(cStart).append("<i>")
+ .append(sentCharEnd).append("<i>")
+ .append(sentEnd).append("<b>1");
+ }
+
+ tv.append("|_").append(tokIdx).append("$<i>")
+ .append(cStart).append("<i>").append(cEnd);
+ tv.append("]");
+
+ charOff = cEnd;
+ tokIdx++;
+ }
+ sentStart = sentEnd;
+ }
+
+ fd.addTV("tokens", surface.toString(), tv.toString());
+ return fd;
+ }
+
+ // focus(1: contains(contains(<s>, {1: s:a}), {2: s:b}))
+ private SpanQuery nestedContainsDiffClassQuery () {
+ SpanQuery sentence = new SpanElementQuery("tokens", "base/s:s");
+ SpanQuery tokenA = new SpanClassQuery(
+ new SpanTermQuery(new Term("tokens", "s:a")), (byte) 1);
+ SpanQuery tokenB = new SpanClassQuery(
+ new SpanTermQuery(new Term("tokens", "s:b")), (byte) 2);
+ SpanQuery innerContains = new SpanWithinQuery(sentence, tokenA);
+ SpanQuery outerContains = new SpanWithinQuery(innerContains, tokenB);
+ SpanFocusQuery focus = new SpanFocusQuery(outerContains, (byte) 1);
+ focus.setSorted(false);
+ return focus;
+ }
+
+ /**
+ * Two docs in one segment where doc 1 has b in a separate sentence.
+ * The fix prevents b from leaking into doc 0's a-only sentence.
+ */
+ @Test
+ public void testNestedContainsSameSegmentCrossDocLeakage ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ // Doc 0: S0 has b+a, S1 has a only.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "b", "a", "w2" },
+ new String[] { "a", "w4", "w5" }));
+
+ // Doc 1: S0 has a only, S1 has b only.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "a", "w1", "w2" },
+ new String[] { "b", "w4", "w5" }));
+
+ ki.commit();
+
+ Result kr = ki.search(nestedContainsDiffClassQuery(), (short) 50);
+ assertEquals("Only Doc 0 S0 has both a+b, expect 1 match",
+ 1, kr.getTotalResults());
+ assertEquals(1, kr.getMatch(0).getStartPos());
+ assertEquals(2, kr.getMatch(0).getEndPos());
+ ki.close();
+ }
+
+ /**
+ * Same data as above but each doc in its own segment.
+ * Verifies no cross-doc leak with separate segments.
+ */
+ @Test
+ public void testNestedContainsSeparateSegmentsNoLeakage ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ // Doc 0: S0 has b+a, S1 has a only.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "b", "a", "w2" },
+ new String[] { "a", "w4", "w5" }));
+ ki.commit();
+
+ // Doc 1: S0 has a only, S1 has b only.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "a", "w1", "w2" },
+ new String[] { "b", "w4", "w5" }));
+ ki.commit();
+
+ Result kr = ki.search(nestedContainsDiffClassQuery(), (short) 50);
+ assertEquals("Separate segments: expect 1 match",
+ 1, kr.getTotalResults());
+ assertEquals(1, kr.getMatch(0).getStartPos());
+ assertEquals(2, kr.getMatch(0).getEndPos());
+ ki.close();
+ }
+
+ /**
+ * Multiple a-only docs then one doc with both a+b in same segment.
+ * B from last doc must not leak into earlier docs.
+ */
+ @Test
+ public void testNestedContainsManyAOnlyDocsThenBoth ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ // Doc 0: single sentence with a only.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "a", "w1", "w2" }));
+ // Doc 1: single sentence with a only.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "w0", "a", "w2" }));
+ // Doc 2: single sentence with a only.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "a", "w1", "w2" }));
+ // Doc 3: single sentence with both a and b.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "a", "b", "w2" }));
+
+ ki.commit();
+
+ Result kr = ki.search(nestedContainsDiffClassQuery(), (short) 50);
+ assertEquals("Only Doc 3 has both, expect 1 match",
+ 1, kr.getTotalResults());
+ assertEquals(0, kr.getMatch(0).getStartPos());
+ assertEquals(1, kr.getMatch(0).getEndPos());
+ ki.close();
+ }
+
+ /**
+ * Multi-sentence a-only doc then doc with a+b in one sentence.
+ * Tests scenario from fuzz seed 125901200207375.
+ */
+ @Test
+ public void testNestedContainsMultiSentenceAOnlyThenBoth ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ // Doc 0: two sentences, both are a only.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "w0", "w1", "a", "w3", "w4" },
+ new String[] { "w5", "a", "w7", "w8", "w9" }));
+
+ // Doc 1: one sentence with both a and b.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "a", "w1", "w2", "b", "w4" }));
+
+ ki.commit();
+
+ Result kr = ki.search(nestedContainsDiffClassQuery(), (short) 50);
+ assertEquals("Only Doc 1 has both a+b, expect 1 match",
+ 1, kr.getTotalResults());
+ assertEquals(0, kr.getMatch(0).getStartPos());
+ assertEquals(1, kr.getMatch(0).getEndPos());
+ ki.close();
+ }
+
+ // Regression: skipTo() must also work before first next().
+ @Test
+ public void testWithinSkipToBeforeNextInitializesSpans ()
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+
+ // One local doc with a single sentence that contains token a.
+ ki.addDoc(multiSentenceDoc(
+ new String[] { "a", "w1", "w2" }));
+ ki.commit();
+
+ SpanQuery sq = new SpanWithinQuery(
+ new SpanElementQuery("tokens", "base/s:s"),
+ new SpanTermQuery(new Term("tokens", "s:a")));
+
+ Map<Term, TermContext> termContexts = new HashMap<>();
+ for (LeafReaderContext atomic : ki.reader().leaves()) {
+ Bits bitset = atomic.reader().getLiveDocs();
+ Spans spans = sq.getSpans(atomic, bitset, termContexts);
+ assertTrue("skipTo() before next() must initialize spans",
+ spans.skipTo(0));
+ assertEquals(0, spans.doc());
+ assertEquals(0, spans.start());
+ assertEquals(3, spans.end());
+ }
+
+ ki.close();
+ }
+
+ // ----------------------------------------------------------------------
+ // Fuzzing tests for nested contains + focus
+ //
+ // Call with:
+ // $ mvn test -Dtest="TestWithinIndex" -Dfuzz.iterations=200 -pl .
+ // ----------------------------------------------------------------------
+
+ // Read iteration count; default 0 keeps fuzz tests disabled.
+ private static int getFuzzIterations () {
+ String prop = System.getProperty("fuzz.iterations");
+ if (prop != null) return Integer.parseInt(prop);
+ return 0;
+ }
+
+ // Read optional seed; default uses current nano time.
+ private static long getFuzzBaseSeed () {
+ String prop = System.getProperty("fuzz.seed");
+ if (prop != null) return Long.parseLong(prop);
+ return System.nanoTime();
+ }
+
+ enum BiasStrategy {
+ UNIFORM,
+ HEAVY_A_RARE_B,
+ ALTERNATING,
+ CLUSTER,
+ DENSE
+ }
+
+ static class SentenceInfo {
+ final int startPos;
+ final int endPos;
+ final List<Integer> aPositions;
+ final List<Integer> bPositions;
+
+ SentenceInfo (int startPos, int endPos,
+ List<Integer> aPositions, List<Integer> bPositions) {
+ this.startPos = startPos;
+ this.endPos = endPos;
+ this.aPositions = aPositions;
+ this.bPositions = bPositions;
+ }
+
+ boolean hasA () { return !aPositions.isEmpty(); }
+ boolean hasB () { return !bPositions.isEmpty(); }
+ boolean hasBoth () { return hasA() && hasB(); }
+ }
+
+ static class DocInfo {
+ final String textSigle;
+ final String genre;
+ final List<SentenceInfo> sentences;
+
+ DocInfo (String textSigle, String genre,
+ List<SentenceInfo> sentences) {
+ this.textSigle = textSigle;
+ this.genre = genre;
+ this.sentences = sentences;
+ }
+ }
+
+ static class IndexConfig {
+ final List<DocInfo> docs;
+ final int[] commitAfterDoc;
+ final long seed;
+ final BiasStrategy strategy;
+
+ IndexConfig (List<DocInfo> docs, int[] commitAfterDoc,
+ long seed, BiasStrategy strategy) {
+ this.docs = docs;
+ this.commitAfterDoc = commitAfterDoc;
+ this.seed = seed;
+ this.strategy = strategy;
+ }
+ }
+
+ // Generate one random fuzz corpus with a random bias strategy.
+ private IndexConfig generateFuzzConfig (long seed) {
+ Random rng = new Random(seed);
+ BiasStrategy[] strategies = BiasStrategy.values();
+ BiasStrategy strategy = strategies[rng.nextInt(strategies.length)];
+ return generateFuzzConfig(seed, rng, strategy);
+ }
+
+ // Generate one random fuzz corpus using a fixed strategy.
+ private IndexConfig generateFuzzConfig (long seed, Random rng,
+ BiasStrategy strategy) {
+ int numDocs = 1 + rng.nextInt(5);
+ List<DocInfo> docs = new ArrayList<>();
+
+ for (int d = 0; d < numDocs; d++) {
+ int numSentences = 2 + rng.nextInt(14);
+ List<SentenceInfo> sentences = new ArrayList<>();
+ int docTokenPos = 0;
+
+ for (int s = 0; s < numSentences; s++) {
+ int sentLen = decideSentenceLength(rng, strategy);
+ int sentStart = docTokenPos;
+ int sentEnd = docTokenPos + sentLen;
+
+ boolean placeA = decidePlaceA(rng, strategy, s, numSentences);
+ boolean placeB = decidePlaceB(rng, strategy, s, numSentences);
+
+ int numA = placeA ? (1 + rng.nextInt(
+ strategy == BiasStrategy.DENSE ? 3 : 2)) : 0;
+ int numB = placeB ? (1 + rng.nextInt(
+ strategy == BiasStrategy.DENSE ? 3 : 2)) : 0;
+ numA = Math.min(numA, sentLen / 2);
+ numB = Math.min(numB, Math.max(0, sentLen - numA));
+
+ Set<Integer> usedOffsets = new HashSet<>();
+ List<Integer> aPositions = new ArrayList<>();
+ List<Integer> bPositions = new ArrayList<>();
+
+ for (int i = 0; i < numA; i++) {
+ int off = pickUnusedOffset(rng, sentLen, usedOffsets);
+ if (off >= 0) {
+ usedOffsets.add(off);
+ aPositions.add(sentStart + off);
+ }
+ }
+ for (int i = 0; i < numB; i++) {
+ int off = pickUnusedOffset(rng, sentLen, usedOffsets);
+ if (off >= 0) {
+ usedOffsets.add(off);
+ bPositions.add(sentStart + off);
+ }
+ }
+
+ sentences.add(new SentenceInfo(sentStart, sentEnd,
+ aPositions, bPositions));
+ docTokenPos = sentEnd;
+ }
+
+ String genre = (d % 3 == 0) ? "science"
+ : (d % 3 == 1) ? "fiction" : "news";
+ docs.add(new DocInfo("TST/D" + d + "/T0", genre, sentences));
+ }
+
+ int[] commitAfterDoc = decideCommitPoints(rng, numDocs);
+ return new IndexConfig(docs, commitAfterDoc, seed, strategy);
+ }
+
+ // Choose sentence length for one random sentence.
+ private int decideSentenceLength (Random rng, BiasStrategy strategy) {
+ if (strategy == BiasStrategy.DENSE) {
+ return 6 + rng.nextInt(10);
+ }
+ int r = rng.nextInt(100);
+ if (r < 5) return 2 + rng.nextInt(2);
+ if (r < 15) return 20 + rng.nextInt(11);
+ return 3 + rng.nextInt(15);
+ }
+
+ // Decide if sentence gets at least one a token.
+ private boolean decidePlaceA (Random rng, BiasStrategy strategy,
+ int sentIdx, int numSentences) {
+ switch (strategy) {
+ case HEAVY_A_RARE_B:
+ return rng.nextDouble() < 0.9;
+ case ALTERNATING:
+ return sentIdx % 2 == 0 || rng.nextDouble() < 0.15;
+ case CLUSTER:
+ return sentIdx < numSentences / 2 || rng.nextDouble() < 0.2;
+ case DENSE:
+ return true;
+ default:
+ return rng.nextDouble() < 0.6;
+ }
+ }
+
+ // Decide if sentence gets at least one b token.
+ private boolean decidePlaceB (Random rng, BiasStrategy strategy,
+ int sentIdx, int numSentences) {
+ switch (strategy) {
+ case HEAVY_A_RARE_B:
+ return rng.nextDouble() < 0.1;
+ case ALTERNATING:
+ return sentIdx % 2 == 1 || rng.nextDouble() < 0.15;
+ case CLUSTER:
+ return sentIdx >= numSentences / 2 || rng.nextDouble() < 0.2;
+ case DENSE:
+ return true;
+ default:
+ return rng.nextDouble() < 0.4;
+ }
+ }
+
+ // Pick a random token offset not used yet in sentence.
+ private int pickUnusedOffset (Random rng, int sentLen,
+ Set<Integer> used) {
+ if (used.size() >= sentLen) return -1;
+ for (int attempt = 0; attempt < 20; attempt++) {
+ int pos = rng.nextInt(sentLen);
+ if (!used.contains(pos)) return pos;
+ }
+ for (int pos = 0; pos < sentLen; pos++) {
+ if (!used.contains(pos)) return pos;
+ }
+ return -1;
+ }
+
+ // Choose where to commit so we get multiple segment layouts.
+ private int[] decideCommitPoints (Random rng, int numDocs) {
+ if (numDocs <= 1) return new int[] { 0 };
+ List<Integer> points = new ArrayList<>();
+ for (int d = 0; d < numDocs; d++) {
+ if (d == numDocs - 1 || rng.nextDouble() < 0.4) {
+ points.add(d);
+ }
+ }
+ if (!points.contains(numDocs - 1)) points.add(numDocs - 1);
+ int[] result = new int[points.size()];
+ for (int i = 0; i < result.length; i++) result[i] = points.get(i);
+ return result;
+ }
+
+ // Build a KrillIndex from generated docs and commit plan.
+ private KrillIndex buildFuzzIndex (IndexConfig config)
+ throws IOException {
+ KrillIndex ki = new KrillIndex();
+ int commitIdx = 0;
+
+ for (int d = 0; d < config.docs.size(); d++) {
+ DocInfo doc = config.docs.get(d);
+ FieldDocument fd = buildFuzzFieldDocument(doc);
+ ki.addDoc(fd);
+
+ if (commitIdx < config.commitAfterDoc.length
+ && config.commitAfterDoc[commitIdx] == d) {
+ ki.commit();
+ commitIdx++;
+ }
+ }
+
+ if (commitIdx == 0 || config.commitAfterDoc[commitIdx - 1]
+ != config.docs.size() - 1) {
+ ki.commit();
+ }
+
+ return ki;
+ }
+
+ // Convert one generated doc into a tokens field with TV data.
+ private FieldDocument buildFuzzFieldDocument (DocInfo doc) {
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", doc.textSigle);
+ fd.addString("genre", doc.genre);
+
+ Set<Integer> aPositions = new HashSet<>();
+ Set<Integer> bPositions = new HashSet<>();
+ for (SentenceInfo sent : doc.sentences) {
+ aPositions.addAll(sent.aPositions);
+ bPositions.addAll(sent.bPositions);
+ }
+
+ int totalTokens = 0;
+ for (SentenceInfo sent : doc.sentences) {
+ totalTokens += (sent.endPos - sent.startPos);
+ }
+
+ String[] allTokens = new String[totalTokens];
+ int idx = 0;
+ for (SentenceInfo sent : doc.sentences) {
+ for (int pos = sent.startPos; pos < sent.endPos; pos++) {
+ if (aPositions.contains(pos)) {
+ allTokens[idx] = "a";
+ }
+ else if (bPositions.contains(pos)) {
+ allTokens[idx] = "b";
+ }
+ else {
+ allTokens[idx] = "w" + pos;
+ }
+ idx++;
+ }
+ }
+
+ int totalChars = 0;
+ for (String t : allTokens) totalChars += t.length();
+
+ StringBuilder surface = new StringBuilder();
+ StringBuilder tv = new StringBuilder();
+
+ int tokIdx = 0;
+ int charOff = 0;
+
+ for (int si = 0; si < doc.sentences.size(); si++) {
+ SentenceInfo sent = doc.sentences.get(si);
+ int sentLen = sent.endPos - sent.startPos;
+
+ int sentCharStart = charOff;
+ int sentCharEnd = sentCharStart;
+ for (int ti = 0; ti < sentLen; ti++) {
+ sentCharEnd += allTokens[tokIdx + ti].length();
+ }
+
+ for (int ti = 0; ti < sentLen; ti++) {
+ String tok = allTokens[tokIdx + ti];
+ int cStart = charOff;
+ int cEnd = charOff + tok.length();
+ surface.append(tok);
+
+ tv.append("[(").append(cStart).append("-").append(cEnd)
+ .append(")");
+ tv.append("s:").append(tok);
+
+ if (tokIdx + ti == 0) {
+ tv.append("|-:t$<i>").append(totalTokens);
+ tv.append("|<>:base/s:t$<b>64<i>0<i>")
+ .append(totalChars).append("<i>")
+ .append(totalTokens).append("<b>0");
+ }
+
+ if (ti == 0) {
+ tv.append("|<>:base/s:s$<b>64<i>")
+ .append(sentCharStart).append("<i>")
+ .append(sentCharEnd).append("<i>")
+ .append(sent.endPos).append("<b>1");
+ }
+
+ tv.append("|_").append(tokIdx + ti).append("$<i>")
+ .append(cStart).append("<i>").append(cEnd);
+ tv.append("]");
+
+ charOff = cEnd;
+ }
+ tokIdx += sentLen;
+ }
+
+ fd.addTV("tokens", surface.toString(), tv.toString());
+ return fd;
+ }
+
+ // focus(1: contains(contains(<s>, {1: a}), {1: b}))
+ private SpanQuery nestedContainsSameClassQuery () {
+ SpanQuery sentence = new SpanElementQuery("tokens", "base/s:s");
+ SpanQuery tokenA = new SpanClassQuery(
+ new SpanTermQuery(new Term("tokens", "s:a")), (byte) 1);
+ SpanQuery tokenB = new SpanClassQuery(
+ new SpanTermQuery(new Term("tokens", "s:b")), (byte) 1);
+ SpanQuery innerContains = new SpanWithinQuery(sentence, tokenA);
+ SpanQuery outerContains = new SpanWithinQuery(innerContains, tokenB);
+ SpanFocusQuery focus = new SpanFocusQuery(outerContains, (byte) 1);
+ focus.setSorted(false);
+ return focus;
+ }
+
+ // Collect all a positions from sentences that also contain b.
+ private Set<Integer> expectedAPositions (DocInfo doc) {
+ Set<Integer> result = new TreeSet<>();
+ for (SentenceInfo sent : doc.sentences) {
+ if (sent.hasBoth()) {
+ result.addAll(sent.aPositions);
+ }
+ }
+ return result;
+ }
+
+ // Collect all a positions in one generated document.
+ private Set<Integer> allAPositions (DocInfo doc) {
+ Set<Integer> result = new TreeSet<>();
+ for (SentenceInfo sent : doc.sentences) {
+ result.addAll(sent.aPositions);
+ }
+ return result;
+ }
+
+ // Collect all a and b positions in one generated document.
+ private Set<Integer> allABPositions (DocInfo doc) {
+ Set<Integer> result = new TreeSet<>();
+ for (SentenceInfo sent : doc.sentences) {
+ result.addAll(sent.aPositions);
+ result.addAll(sent.bPositions);
+ }
+ return result;
+ }
+
+ // Check invariants for focus(class1) over different class ids.
+ private void verifyDiffClassInvariants (Result kr,
+ IndexConfig config, String context) {
+
+ Set<Integer> anyDocExpectedA = new TreeSet<>();
+ for (DocInfo doc : config.docs) {
+ anyDocExpectedA.addAll(expectedAPositions(doc));
+ }
+
+ Set<Integer> matchedPositions = new TreeSet<>();
+ int storedMatches = kr.getMatches().size();
+
+ for (int i = 0; i < storedMatches; i++) {
+ Match m = kr.getMatch(i);
+ int mStart = m.getStartPos();
+ int mEnd = m.getEndPos();
+
+ assertTrue(context + ": Match " + i + " [" + mStart + ","
+ + mEnd + ") must be single token (end==start+1)",
+ mEnd == mStart + 1);
+
+ boolean validInSomeDoc = false;
+ for (DocInfo doc : config.docs) {
+ if (allAPositions(doc).contains(mStart)
+ && expectedAPositions(doc).contains(mStart)) {
+ validInSomeDoc = true;
+ break;
+ }
+ }
+ assertTrue(context + ": Match " + i + " start=" + mStart
+ + " must be a valid 'a' in a sentence with 'b'",
+ validInSomeDoc);
+
+ matchedPositions.add(mStart);
+ }
+
+ if (config.docs.size() == 1
+ && kr.getTotalResults() == storedMatches) {
+ Set<Integer> expected = expectedAPositions(config.docs.get(0));
+ for (int pos : expected) {
+ assertTrue(context + ": Expected 'a' at position "
+ + pos + " not found in results",
+ matchedPositions.contains(pos));
+ }
+ }
+ }
+
+ // Check invariants for focus(class1) with shared class ids.
+ private void verifySameClassInvariants (Result kr,
+ IndexConfig config, String context) {
+
+ int storedMatches = kr.getMatches().size();
+ for (int i = 0; i < storedMatches; i++) {
+ Match m = kr.getMatch(i);
+ int mStart = m.getStartPos();
+ int mEnd = m.getEndPos();
+
+ boolean validInSomeDoc = false;
+ for (DocInfo doc : config.docs) {
+ Set<Integer> abPos = allABPositions(doc);
+
+ if (!abPos.contains(mStart)) continue;
+ if (!abPos.contains(mEnd - 1)) continue;
+
+ boolean withinSentence = false;
+ boolean sentHasBoth = false;
+ for (SentenceInfo sent : doc.sentences) {
+ if (mStart >= sent.startPos && mEnd <= sent.endPos) {
+ withinSentence = true;
+ sentHasBoth = sent.hasBoth();
+ break;
+ }
+ }
+ if (withinSentence && sentHasBoth) {
+ validInSomeDoc = true;
+ break;
+ }
+ }
+
+ assertTrue(context + ": Match " + i + " [" + mStart + ","
+ + mEnd + ") failed same-class invariants",
+ validInSomeDoc);
+ }
+ }
+
+ // Check diff-class invariants after virtual corpus filtering.
+ private void verifyDiffClassInvariantsFiltered (Result kr,
+ IndexConfig config, Set<String> includedSigles,
+ String context) {
+
+ int storedMatches = kr.getMatches().size();
+ for (int i = 0; i < storedMatches; i++) {
+ Match m = kr.getMatch(i);
+ int mStart = m.getStartPos();
+ int mEnd = m.getEndPos();
+
+ assertTrue(context + ": Match " + i + " [" + mStart + ","
+ + mEnd + ") must be single token",
+ mEnd == mStart + 1);
+
+ boolean validInFilteredDoc = false;
+ for (DocInfo doc : config.docs) {
+ if (!includedSigles.contains(doc.textSigle)) continue;
+ if (expectedAPositions(doc).contains(mStart)) {
+ validInFilteredDoc = true;
+ break;
+ }
+ }
+ assertTrue(context + ": Match " + i + " start=" + mStart
+ + " must be valid 'a' in filtered docs",
+ validInFilteredDoc);
+ }
+ }
+
+ // Function type used by fuzzLoop for one seeded iteration.
+ @FunctionalInterface
+ interface FuzzAction {
+ void run (long seed) throws Exception;
+ }
+
+ // Run a fuzz action over N seeds and print reproducible failures.
+ private void fuzzLoop (String testName, FuzzAction action) {
+ int iterations = getFuzzIterations();
+ assumeTrue("Fuzz tests skipped (set -Dfuzz.iterations=N to run)",
+ iterations > 0);
+ long baseSeed = getFuzzBaseSeed();
+
+ for (int i = 0; i < iterations; i++) {
+ long seed = baseSeed + i;
+ try {
+ action.run(seed);
+ }
+ catch (AssertionError e) {
+ throw new AssertionError(
+ "FUZZ FAILURE [" + testName + "] iteration="
+ + i + " seed=" + seed + "\n"
+ + "Reproduce: -Dfuzz.seed=" + seed
+ + " -Dfuzz.iterations=1", e);
+ }
+ catch (Exception e) {
+ throw new AssertionError(
+ "FUZZ ERROR [" + testName + "] iteration="
+ + i + " seed=" + seed, e);
+ }
+ }
+ }
+
+ // Fuzz test: class1 focus with inner class1 and outer class2.
+ @Test
+ public void fuzzNestedContainsDiffClass () {
+ fuzzLoop("fuzzNestedContainsDiffClass", seed -> {
+ IndexConfig config = generateFuzzConfig(seed);
+ KrillIndex ki = buildFuzzIndex(config);
+ try {
+ Result kr = ki.search(
+ nestedContainsDiffClassQuery(), (short) 500);
+ verifyDiffClassInvariants(kr, config,
+ "seed=" + seed + " strategy=" + config.strategy);
+ }
+ finally {
+ ki.close();
+ }
+ });
+ }
+
+ // Fuzz test: class1 focus with both terms in class1.
+ @Test
+ public void fuzzNestedContainsSameClass () {
+ fuzzLoop("fuzzNestedContainsSameClass", seed -> {
+ IndexConfig config = generateFuzzConfig(seed);
+ KrillIndex ki = buildFuzzIndex(config);
+ try {
+ Result kr = ki.search(
+ nestedContainsSameClassQuery(), (short) 500);
+ verifySameClassInvariants(kr, config,
+ "seed=" + seed + " strategy=" + config.strategy);
+ }
+ finally {
+ ki.close();
+ }
+ });
+ }
+
+ // Fuzz test: force one commit per doc to maximize segment count.
+ @Test
+ public void fuzzNestedContainsMultiSegment () {
+ fuzzLoop("fuzzNestedContainsMultiSegment", seed -> {
+ Random rng = new Random(seed);
+ BiasStrategy strategy = BiasStrategy.values()[
+ rng.nextInt(BiasStrategy.values().length)];
+ IndexConfig config = generateFuzzConfig(seed, rng, strategy);
+
+ int numDocs = config.docs.size();
+ int[] manyCommits = new int[numDocs];
+ for (int d = 0; d < numDocs; d++) manyCommits[d] = d;
+ IndexConfig multiSegConfig = new IndexConfig(
+ config.docs, manyCommits, seed, strategy);
+
+ KrillIndex ki = buildFuzzIndex(multiSegConfig);
+ try {
+ Result kr = ki.search(
+ nestedContainsDiffClassQuery(), (short) 500);
+ verifyDiffClassInvariants(kr, multiSegConfig,
+ "seed=" + seed + " strategy=" + strategy);
+ }
+ finally {
+ ki.close();
+ }
+ });
+ }
+
+ // Fuzz test: run diff-class query on a filtered virtual corpus.
+ @Test
+ public void fuzzNestedContainsVirtualCorpus () {
+ fuzzLoop("fuzzNestedContainsVirtualCorpus", seed -> {
+ IndexConfig config = generateFuzzConfig(seed);
+ if (config.docs.size() < 2) return;
+
+ KrillIndex ki = buildFuzzIndex(config);
+ try {
+ String targetSigle = config.docs.get(0).textSigle;
+ Set<String> included = new HashSet<>();
+ included.add(targetSigle);
+
+ Krill ks = new Krill(nestedContainsDiffClassQuery());
+ ks.getMeta().setCount((short) 500);
+
+ CollectionBuilder cb = new CollectionBuilder();
+ KrillCollection kc = new KrillCollection(ki);
+ kc.fromBuilder(cb.term("textSigle", targetSigle));
+ ks.setCollection(kc);
+
+ Result kr = ks.apply(ki);
+ verifyDiffClassInvariantsFiltered(kr, config, included,
+ "seed=" + seed + " vc_sigle=" + targetSigle);
+ }
+ finally {
+ ki.close();
+ }
+ });
+ }
+
+ // Fuzz test: heavy backtracking setup with many a and rare b.
+ @Test
+ public void fuzzNestedContainsHeavyBacktracking () {
+ fuzzLoop("fuzzNestedContainsHeavyBacktracking", seed -> {
+ Random rng = new Random(seed);
+ IndexConfig config = generateFuzzConfig(seed, rng,
+ BiasStrategy.HEAVY_A_RARE_B);
+ KrillIndex ki = buildFuzzIndex(config);
+ try {
+ Result kr = ki.search(
+ nestedContainsDiffClassQuery(), (short) 500);
+ verifyDiffClassInvariants(kr, config,
+ "seed=" + seed + " strategy=HEAVY_A_RARE_B");
+ }
+ finally {
+ ki.close();
+ }
+ });
+ }
+
+ // Fuzz test: dense setup where almost every sentence has a and b.
+ @Test
+ public void fuzzNestedContainsDense () {
+ fuzzLoop("fuzzNestedContainsDense", seed -> {
+ Random rng = new Random(seed);
+ IndexConfig config = generateFuzzConfig(seed, rng,
+ BiasStrategy.DENSE);
+ KrillIndex ki = buildFuzzIndex(config);
+ try {
+ Result kr = ki.search(
+ nestedContainsDiffClassQuery(), (short) 500);
+ verifyDiffClassInvariants(kr, config,
+ "seed=" + seed + " strategy=DENSE");
+ }
+ finally {
+ ki.close();
+ }
+ });
+ }
};
diff --git a/src/test/java/de/ids_mannheim/korap/search/TestVcField.java b/src/test/java/de/ids_mannheim/korap/search/TestVcField.java
index 1a4ed72..f843bef 100644
--- a/src/test/java/de/ids_mannheim/korap/search/TestVcField.java
+++ b/src/test/java/de/ids_mannheim/korap/search/TestVcField.java
@@ -24,6 +24,8 @@
FieldDocument fd = new FieldDocument();
fd.addString("textSigle", textSigle);
fd.setUID(uid);
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
return fd;
}