Ignore tokens in meta field collections
Change-Id: Iaaf085bfb2c6850c92c3af3c8c624c47ddea6344
diff --git a/src/main/java/de/ids_mannheim/korap/KrillIndex.java b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
index ad46ec3..d3d507a 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
@@ -1589,6 +1589,9 @@
while (fieldIterator.hasNext()) {
IndexableField iField = fieldIterator.next();
+ if (iField.name().equals("tokens"))
+ continue;
+
// Add field
metaFields.add(iField);
};
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
index 3dadfcc..0b3498d 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
@@ -242,5 +242,31 @@
assertEquals("koral:field", res.at("/document/fields/5/@type").asText());
assertEquals("Peter", res.at("/document/fields/5/value").asText());
+
+ // Test with real document
+ ki.addDoc(getClass().getResourceAsStream("/wiki/wdd17-982-72848.json.gz"),true);
+
+ /* Save documents */
+ ki.commit();
+
+ res = ki.getFields("wdd17/982/72841").toJsonNode();
+
+ assertEquals("Document not found", res.at("/errors/0/1").asText());
+
+ res = ki.getFields("WDD17/982/72848").toJsonNode();
+
+ assertEquals("type:number", res.at("/document/fields/0/type").asText());
+ assertEquals("pubDate", res.at("/document/fields/0/key").asText());
+ assertEquals(20170701, res.at("/document/fields/0/value").asInt());
+
+ assertEquals("type:string", res.at("/document/fields/1/type").asText());
+ assertEquals("textSigle", res.at("/document/fields/1/key").asText());
+ assertEquals("WDD17/982/72848", res.at("/document/fields/1/value").asText());
+
+ // TODO:
+ // This should better be an array!
+ assertEquals("type:string", res.at("/document/fields/2/type").asText());
+ assertEquals("foundries", res.at("/document/fields/2/key").asText());
+ assertEquals("dereko dereko/structure dereko/structure/base-sentences-paragraphs-pagebreaks lwc lwc/dependency treetagger treetagger/morpho", res.at("/document/fields/2/value").asText());
};
};