Prevent indexation of metadata-only documents
Change-Id: I3e8ab35944837b40f63e54b98a9c7cab84808b70
diff --git a/Changes b/Changes
index 3cf53bb..480e77e 100644
--- a/Changes
+++ b/Changes
@@ -1,7 +1,8 @@
-0.65 2026-05-12
-includes all changes in 0.64.7
+0.65.1 2026-06-10
+ - [feature] Prevent indexation of documents without
+ a token stream (diewald; AI-assisted Claude Opus 4.6)
-0.64.7 2026-05-07
+0.65 2026-05-12
- [bugfix] Keep highlights that extend beyond a cut match
(diewald; fixes #177; diewald; AI-assisted Claude Opus 4.6)
- [bugfix] Correctly handle foundry and layer in attribute groups
diff --git a/pom.xml b/pom.xml
index 287e281..36beae8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -35,7 +35,7 @@
<groupId>de.ids-mannheim.korap.krill</groupId>
<artifactId>Krill</artifactId>
- <version>0.65</version>
+ <version>0.65.1</version>
<packaging>jar</packaging>
<name>Krill</name>
diff --git a/src/main/java/de/ids_mannheim/korap/KrillIndex.java b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
index 0bcd1cd..0b673e9 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
@@ -496,6 +496,15 @@
if (doc == null)
return doc;
+ if (!hasTokenField(doc)) {
+ log.error(
+ "Rejecting upsert for document '{}': no token stream - "
+ + "existing document (if any) will not be removed",
+ doc.getTextSigle()
+ );
+ return doc;
+ }
+
// Create a filter based on the corpusID and the docID
String textSigle = doc.getTextSigle();
KrillDate current = new KrillDate(LocalDate.now());
@@ -608,6 +617,26 @@
/**
+ * Check if a FieldDocument has a known token stream field.
+ * Checks for "tokens" first (current KorAP-XML-Krill format),
+ * then falls back to "base" (legacy format).
+ * A document without either field will not contribute
+ * to token/sentence/paragraph statistics and cannot be searched.
+ *
+ * @param doc The FieldDocument to check
+ * @return true if the document has a known token field with term vectors
+ */
+ public boolean hasTokenField (FieldDocument doc) {
+ if (doc == null)
+ return false;
+ IndexableField field = doc.doc.getField("tokens");
+ if (field == null)
+ field = doc.doc.getField("base");
+ return field != null && field.fieldType().storeTermVectors();
+ }
+
+
+ /**
* Add a document to the index as a {@link FieldDocument}.
*
* @param doc
@@ -619,6 +648,16 @@
if (doc == null)
return doc;
+ if (!hasTokenField(doc)) {
+ log.error(
+ "Rejecting document '{}': no token stream - "
+ + "the document would not contribute to statistics "
+ + "and cannot be searched",
+ doc.getTextSigle()
+ );
+ return doc;
+ }
+
try {
// Add document to writer
diff --git a/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionIndex.java b/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionIndex.java
index 8b5fcef..eaaba0a 100644
--- a/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/collection/TestKrillCollectionIndex.java
@@ -1296,6 +1296,8 @@
for (int i = 0; i < 6000; i++) {
FieldDocument fd = new FieldDocument();
fd.addString("UID", Integer.toString(i));
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
if (i == 4500)
ki.commit();
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java b/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
index a7b3777..f4b3c2c 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestFieldDocument.java
@@ -637,6 +637,245 @@
};
+ /**
+ * A document with only metadata (no token stream / no "data" section)
+ * should be rejected by addDoc with an error.
+ */
+ @Test
+ public void testAddDocRejectsMetadataOnlyDocument () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/META/001");
+ fd.addString("author", "Test Author");
+ fd.addDate("pubDate", 20200101);
+
+ FieldDocument result = ki.addDoc(fd);
+ ki.commit();
+
+ assertEquals(0, ki.numberOf("documents"));
+ assertTrue(
+ "addDoc should return the doc even when rejected",
+ result != null
+ );
+ }
+
+ /**
+ * A document with only metadata should also be rejected via upsertDoc.
+ */
+ @Test
+ public void testUpsertDocRejectsMetadataOnlyDocument () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/META/002");
+ fd.addString("author", "Test Author");
+
+ FieldDocument result = ki.upsertDoc(fd);
+ ki.commit();
+
+ assertEquals(0, ki.numberOf("documents"));
+ }
+
+ /**
+ * A JSON document with fields but no data section should be rejected.
+ */
+ @Test
+ public void testAddDocRejectsJsonWithoutTokenStream () throws Exception {
+ String json = "{"
+ + " \"fields\" : ["
+ + " {"
+ + " \"@type\" : \"koral:field\","
+ + " \"type\" : \"type:string\","
+ + " \"key\" : \"textSigle\","
+ + " \"value\" : \"TEST/NODATA/001\""
+ + " },"
+ + " {"
+ + " \"@type\" : \"koral:field\","
+ + " \"type\" : \"type:text\","
+ + " \"key\" : \"author\","
+ + " \"value\" : \"Nobody\""
+ + " },"
+ + " {"
+ + " \"@type\" : \"koral:field\","
+ + " \"type\" : \"type:date\","
+ + " \"key\" : \"pubDate\","
+ + " \"value\" : \"2020-01-01\""
+ + " }"
+ + " ]"
+ + "}";
+
+ KrillIndex ki = new KrillIndex();
+ FieldDocument fd = ki.addDoc(json);
+ ki.commit();
+
+ assertEquals(
+ "Metadata-only document should not be indexed",
+ 0, ki.numberOf("documents")
+ );
+ }
+
+ /**
+ * A document with a token stream on an unknown field name
+ * (neither "tokens" nor "base") should be rejected.
+ */
+ @Test
+ public void testAddDocRejectsUnknownTokenFieldName () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/WRONG/001");
+ fd.addTV("other", "hello world",
+ "[(0-5)s:hello|i:hello|_0$<i>0<i>5|-:tokens$<i>2]"
+ + "[(6-11)s:world|i:world|_1$<i>6<i>11]");
+ ki.addDoc(fd);
+ ki.commit();
+
+ assertEquals(
+ "Document with token stream on unknown field should be rejected",
+ 0, ki.numberOf("documents")
+ );
+ }
+
+ /**
+ * A document with a token stream on the legacy "base" field
+ * should be accepted.
+ */
+ @Test
+ public void testAddDocAcceptsLegacyBaseField () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ FieldDocument fd = new FieldDocument();
+ fd.addString("textSigle", "TEST/BASE/001");
+ fd.addTV("base", "hello world",
+ "[(0-5)s:hello|i:hello|_0$<i>0<i>5|-:t$<i>2]"
+ + "[(6-11)s:world|i:world|_1$<i>6<i>11]");
+ ki.addDoc(fd);
+ ki.commit();
+
+ assertEquals(
+ "Document with legacy 'base' field should be accepted",
+ 1, ki.numberOf("documents")
+ );
+ }
+
+ /**
+ * When metadata-only documents are mixed with proper documents,
+ * only proper documents should be indexed and contribute to stats.
+ */
+ @Test
+ public void testStatsWithMixedValidAndMetadataOnlyDocuments () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ // Valid document with token stream
+ FieldDocument fd1 = new FieldDocument();
+ fd1.addString("textSigle", "TEST/OK/001");
+ fd1.addString("author", "Good Author");
+ fd1.addTV("tokens", "hello world",
+ "[(0-5)s:hello|i:hello|_0$<i>0<i>5|-:tokens$<i>2]"
+ + "[(6-11)s:world|i:world|_1$<i>6<i>11]");
+ ki.addDoc(fd1);
+
+ // Metadata-only document (should be rejected)
+ FieldDocument fd2 = new FieldDocument();
+ fd2.addString("textSigle", "TEST/BAD/001");
+ fd2.addString("author", "Bad Author");
+ ki.addDoc(fd2);
+
+ // Another valid document
+ FieldDocument fd3 = new FieldDocument();
+ fd3.addString("textSigle", "TEST/OK/002");
+ fd3.addString("author", "Another Author");
+ fd3.addTV("tokens", "foo bar baz",
+ "[(0-3)s:foo|i:foo|_0$<i>0<i>3|-:tokens$<i>3]"
+ + "[(4-7)s:bar|i:bar|_1$<i>4<i>7]"
+ + "[(8-11)s:baz|i:baz|_2$<i>8<i>11]");
+ ki.addDoc(fd3);
+
+ ki.commit();
+
+ assertEquals(
+ "Only valid documents should be counted",
+ 2, ki.numberOf("documents")
+ );
+ assertEquals(5, ki.numberOf("tokens"));
+ }
+
+ /**
+ * A document with an empty stream (data section present but stream
+ * is empty) has a token stream field but no tokens. This is allowed
+ * through indexing since the document structure is valid (it has a
+ * "data" section), but it contributes 0 to token statistics.
+ */
+ @Test
+ public void testAddDocAcceptsEmptyTokenStream () throws Exception {
+ String json = "{"
+ + " \"data\" : {"
+ + " \"text\" : \"\","
+ + " \"name\" : \"tokens\","
+ + " \"stream\" : []"
+ + " },"
+ + " \"fields\" : ["
+ + " {"
+ + " \"@type\" : \"koral:field\","
+ + " \"type\" : \"type:string\","
+ + " \"key\" : \"textSigle\","
+ + " \"value\" : \"TEST/EMPTY/001\""
+ + " }"
+ + " ]"
+ + "}";
+
+ KrillIndex ki = new KrillIndex();
+ FieldDocument fd = ki.addDoc(json);
+ ki.commit();
+
+ assertEquals(
+ "Document with empty stream is accepted (has token field)",
+ 1, ki.numberOf("documents")
+ );
+ assertEquals(
+ "Empty stream contributes 0 tokens",
+ 0, ki.numberOf("tokens")
+ );
+ }
+
+ /**
+ * Upserting a valid document with a metadata-only replacement should
+ * NOT remove the original and should reject the new one.
+ */
+ @Test
+ public void testUpsertDoesNotReplaceValidDocWithMetadataOnly () throws Exception {
+ KrillIndex ki = new KrillIndex();
+
+ // First: add a valid document
+ FieldDocument fd1 = new FieldDocument();
+ fd1.addString("textSigle", "TEST/UPSERT/001");
+ fd1.addString("author", "Original");
+ fd1.addTV("tokens", "good data",
+ "[(0-4)s:good|i:good|_0$<i>0<i>4|-:tokens$<i>2]"
+ + "[(5-9)s:data|i:data|_1$<i>5<i>9]");
+ ki.addDoc(fd1);
+ ki.commit();
+
+ assertEquals(1, ki.numberOf("documents"));
+ assertEquals(2, ki.numberOf("tokens"));
+
+ // Now try to upsert with metadata-only (should be rejected)
+ FieldDocument fd2 = new FieldDocument();
+ fd2.addString("textSigle", "TEST/UPSERT/001");
+ fd2.addString("author", "Replacement Without Tokens");
+ ki.upsertDoc(fd2);
+ ki.commit();
+
+ // The original document should still be there since the upsert
+ // was rejected before the delete could happen
+ assertEquals(1, ki.numberOf("documents"));
+ assertEquals(2, ki.numberOf("tokens"));
+
+ MetaFields mfs = ki.getFields("TEST/UPSERT/001");
+ assertEquals("Original", mfs.getFieldValue("author"));
+ }
+
@Test
public void indexUpsert () throws Exception {
KrillIndex ki = new KrillIndex();
@@ -645,6 +884,8 @@
FieldDocument fd = new FieldDocument();
fd.addString("textSigle", "AAA/BBB/001");
fd.addString("content", "Example1");
+ fd.addTV("tokens", "Example1",
+ "[(0-8)s:Example1|i:example1|_0$<i>0<i>8|-:tokens$<i>1]");
ki.upsertDoc(fd);
ki.commit();
@@ -662,6 +903,8 @@
fd = new FieldDocument();
fd.addString("textSigle", "AAA/BBB/002");
fd.addString("content", "Example2");
+ fd.addTV("tokens", "Example2",
+ "[(0-8)s:Example2|i:example2|_0$<i>0<i>8|-:tokens$<i>1]");
ki.upsertDoc(fd);
ki.commit();
@@ -675,6 +918,8 @@
fd = new FieldDocument();
fd.addString("textSigle", "AAA/BBB/001");
fd.addString("content", "Example3");
+ fd.addTV("tokens", "Example3",
+ "[(0-8)s:Example3|i:example3|_0$<i>0<i>8|-:tokens$<i>1]");
ki.upsertDoc(fd);
ki.commit();
@@ -696,6 +941,8 @@
fd = new FieldDocument();
fd.addString("textSigle", "AAA/DDD/005");
fd.addString("content", "Example4");
+ fd.addTV("tokens", "Example4",
+ "[(0-8)s:Example4|i:example4|_0$<i>0<i>8|-:tokens$<i>1]");
ki.upsertDoc(fd);
ki.commit();
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
index e6c7d93..a301544 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
@@ -114,12 +114,16 @@
FieldDocument fd = new FieldDocument();
fd.addString("name", "Peter");
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
assertEquals(0, ki.numberOf("base", "documents"));
fd = new FieldDocument();
fd.addString("name", "Michael");
+ fd.addTV("tokens", "y",
+ "[(0-1)s:y|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
assertEquals(0, ki.numberOf("base", "documents"));
@@ -170,11 +174,15 @@
FieldDocument fd = new FieldDocument();
fd.addText("title", "Peter");
fd.setUID(22);
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
fd = new FieldDocument();
fd.addText("title", "Akron");
fd.setUID("05678");
+ fd.addTV("tokens", "y",
+ "[(0-1)s:y|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
ki.commit();
@@ -211,6 +219,8 @@
fd.addText("title", "Der Name der Rose");
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
/* Save documents */
@@ -335,6 +345,8 @@
FieldDocument fd = new FieldDocument();
fd.addString("name", "Peter");
fd.addString("textSigle", "a/b/c");
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.upsertDoc(fd);
/* Save documents */
@@ -343,6 +355,8 @@
fd = new FieldDocument();
fd.addString("name", "Frank");
fd.addString("textSigle", "a/b/d");
+ fd.addTV("tokens", "y",
+ "[(0-1)s:y|_0$<i>0<i>1|-:tokens$<i>1]");
ki.upsertDoc(fd);
/* Save documents */
@@ -351,6 +365,8 @@
fd = new FieldDocument();
fd.addString("name", "Franz");
fd.addString("textSigle", "a/b/c");
+ fd.addTV("tokens", "z",
+ "[(0-1)s:z|_0$<i>0<i>1|-:tokens$<i>1]");
ki.upsertDoc(fd);
/* Save documents */
@@ -384,11 +400,15 @@
FieldDocument fd = new FieldDocument();
fd.addString("textSigle", "aaaa");
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
fd = new FieldDocument();
fd.addString("textSigle", "bbbb");
fd.setUID("05678");
+ fd.addTV("tokens", "y",
+ "[(0-1)s:y|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
ki.commit();
@@ -413,6 +433,8 @@
fd = new FieldDocument();
fd.addString("textSigle", "cccc");
+ fd.addTV("tokens", "z",
+ "[(0-1)s:z|_0$<i>0<i>1|-:tokens$<i>1]");
ki.addDoc(fd);
ki.commit();
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestPrimaryDataProtection.java b/src/test/java/de/ids_mannheim/korap/index/TestPrimaryDataProtection.java
index 11a6421..191ced2 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestPrimaryDataProtection.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestPrimaryDataProtection.java
@@ -385,6 +385,10 @@
fd.addString("textSigle", "TST-004-0001");
fd.addText("title", "Custom Field Document");
fd.setUID(45);
+ fd.addTV("tokens", "some indexed text",
+ "[(0-4)s:some|_0#0-4|-:t$<i>3]"
+ + "[(5-12)s:indexed|_1#5-12]"
+ + "[(13-17)s:text|_2#13-17]");
fd.addTV("customTokens", "leaked custom text",
"[(0-6)s:leaked|_0#0-6|-:t$<i>3]"
+ "[(7-13)s:custom|_1#7-13]"
diff --git a/src/test/java/de/ids_mannheim/korap/search/TestVcField.java b/src/test/java/de/ids_mannheim/korap/search/TestVcField.java
index 1a4ed72..f843bef 100644
--- a/src/test/java/de/ids_mannheim/korap/search/TestVcField.java
+++ b/src/test/java/de/ids_mannheim/korap/search/TestVcField.java
@@ -24,6 +24,8 @@
FieldDocument fd = new FieldDocument();
fd.addString("textSigle", textSigle);
fd.setUID(uid);
+ fd.addTV("tokens", "x",
+ "[(0-1)s:x|_0$<i>0<i>1|-:tokens$<i>1]");
return fd;
}