Embed MetaFieldsObj in MetaFields
Change-Id: Id12622ada28cc4771b88d5a459c8a172e14cd998
diff --git a/src/main/java/de/ids_mannheim/korap/KrillIndex.java b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
index e47589e..7829387 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
@@ -1632,17 +1632,7 @@
continue;
Document doc = atomic.reader().document(localDocID);
-
- Iterator<IndexableField> fieldIterator = doc.getFields().iterator();
- while (fieldIterator.hasNext()) {
- IndexableField iField = fieldIterator.next();
-
- if (iField.name().equals("tokens"))
- continue;
-
- // Add field
- metaFields.add(iField);
- };
+ metaFields.populateFields(doc);
return metaFields;
};
diff --git a/src/main/java/de/ids_mannheim/korap/index/AbstractDocument.java b/src/main/java/de/ids_mannheim/korap/index/AbstractDocument.java
index 9b8f395..e579a16 100644
--- a/src/main/java/de/ids_mannheim/korap/index/AbstractDocument.java
+++ b/src/main/java/de/ids_mannheim/korap/index/AbstractDocument.java
@@ -6,7 +6,7 @@
import de.ids_mannheim.korap.index.FieldDocument;
import de.ids_mannheim.korap.response.Response;
import de.ids_mannheim.korap.response.MetaField;
-import de.ids_mannheim.korap.response.MetaFieldsExt;
+import de.ids_mannheim.korap.response.MetaFieldsObj;
import org.apache.lucene.document.Document;
import org.apache.lucene.index.IndexableField;
@@ -103,7 +103,7 @@
public int internalDocID, localDocID, UID;
@JsonIgnore
- public MetaFieldsExt mFields = new MetaFieldsExt();
+ public MetaFieldsObj mFields = new MetaFieldsObj();
/**
* Populate document meta information with information coming from
@@ -123,6 +123,23 @@
this.populateDocument(doc, field, fieldList);
};
+ /**
+ * Populate document meta information with information coming from
+ * the index.
+ *
+ * @param doc
+ * Document object.
+ * @param field
+ * Primary data field.
+ * @param fields
+ * Hash object with all supported fields.
+ */
+ public void populateDocument (Document doc, String field,
+ Collection<String> fields) {
+ this.setPrimaryData(doc.get(field));
+ this.populateFields(doc, fields);
+ };
+
public void populateFields (Document doc) {
@@ -162,24 +179,6 @@
};
};
-
-
- /**
- * Populate document meta information with information coming from
- * the index.
- *
- * @param doc
- * Document object.
- * @param field
- * Primary data field.
- * @param fields
- * Hash object with all supported fields.
- */
- public void populateDocument (Document doc, String field,
- Collection<String> fields) {
- this.setPrimaryData(doc.get(field));
- this.populateFields(doc, fields);
- };
/**
diff --git a/src/main/java/de/ids_mannheim/korap/response/MetaFields.java b/src/main/java/de/ids_mannheim/korap/response/MetaFields.java
index 941a137..e083769 100644
--- a/src/main/java/de/ids_mannheim/korap/response/MetaFields.java
+++ b/src/main/java/de/ids_mannheim/korap/response/MetaFields.java
@@ -3,6 +3,9 @@
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.StringReader;
+import java.util.*;
+
import com.fasterxml.jackson.annotation.*;
import com.fasterxml.jackson.annotation.JsonInclude.Include;
import com.fasterxml.jackson.databind.ObjectMapper;
@@ -11,183 +14,21 @@
import com.fasterxml.jackson.databind.node.ArrayNode;
import de.ids_mannheim.korap.index.AbstractDocument;
-import de.ids_mannheim.korap.util.KrillDate;
-
-import java.io.IOException;
-
-import de.ids_mannheim.korap.index.KeywordAnalyzer;
-import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
-
-import java.io.StringReader;
-
-import java.util.*;
-import java.util.regex.*;
import org.apache.lucene.index.*;
+import org.apache.lucene.document.Document;
-@JsonInclude(Include.NON_NULL)
public class MetaFields extends AbstractDocument {
- // Logger
- private final static Logger log = LoggerFactory.getLogger(MetaFields.class);
-
- // This advices the java compiler to ignore all loggings
- public static final boolean DEBUG = false;
-
- // TODO:
- // This is a temporary indicator to check
- // whether a date field is a date
- private static final Pattern dateKeyPattern = Pattern.compile(".*Date$");
-
- // Mapper for JSON serialization
+ // Mapper for JSON serialization
ObjectMapper mapper = new ObjectMapper();
- private Map<String, MetaField> fieldsMap = new HashMap<>();
-
-
- public MetaFields () {};
-
-
- public MetaFields (String id) {
- this.addMessage(0, "Response format is temporary");
- };
-
-
- /**
- * Add field to collection
- */
- public void add (IndexableField iField) {
-
- IndexableFieldType iFieldType = iField.fieldType();
-
- // Field type needs to be restored heuristically
- // - though that's not very elegant
-
- // Ignore non-stored fields
- if (!iFieldType.stored())
- return;
-
- MetaField mf = new MetaField(iField.name());
-
- // Reuse existing metafield
- if (fieldsMap.containsKey(mf.key)) {
- mf = fieldsMap.get(mf.key);
- }
-
- // Add new field
- else {
- fieldsMap.put(mf.key, mf);
- };
-
- // TODO: Check if metaField exists for that field
-
- Number n = iField.numericValue();
- String s = iField.stringValue();
-
- // Field has numeric value (possibly a date)
- if (n != null) {
-
- // Check if key indicates a date
- Matcher dateMatcher = dateKeyPattern.matcher(mf.key);
- if (dateMatcher.matches()) {
- mf.type = "type:date";
-
- // Check structure with KrillDate
- KrillDate date = new KrillDate(n.toString());
- if (date != null) {
-
- // Serialize withz dash separation
- mf.values.add(date.toDisplay());
- };
- }
-
- // Field is a number
- else {
- mf.type = "type:integer";
- mf.values.add(n.toString());
- };
- }
-
- // Field has a textual value
- else if (s != null) {
-
- // Stored
- if (iFieldType.indexOptions() == IndexOptions.NONE) {
- String value = s.toString();
- if (value.startsWith("data:")) {
- mf.type = "type:attachement";
- }
- else {
- mf.type = "type:store";
- };
- mf.values.add(value);
- }
-
- // Keywords
- else if (iFieldType.indexOptions() == IndexOptions.DOCS_AND_FREQS) {
- mf.type = "type:keywords";
-
- // Analyze keywords
- try {
- StringReader reader = new StringReader(s.toString());
- KeywordAnalyzer kwa = new KeywordAnalyzer();
- TokenStream ts = kwa.tokenStream("-", reader);
- CharTermAttribute term;
- ts.reset();
- while (ts.incrementToken()) {
- term = ts.getAttribute(CharTermAttribute.class);
- mf.values.add(term.toString());
- };
- ts.close();
- reader.close();
- }
- catch (IOException e) {
- log.error("Unable to split {}={}", iField.name(), s.toString());
- }
- }
-
- // Text
- else if (iFieldType.indexOptions() != IndexOptions.DOCS) {
- mf.type = "type:text";
- mf.values.add(s.toString());
- }
-
- // Special treatment for legacy indices
- else if (mf.key.equals("UID")) {
- mf.type = "type:integer";
- mf.values.add(s.toString());
- }
-
- // String
- else {
- mf.values.add(s.toString());
- };
- }
-
- else {
- log.error("Unknown field type {}", iField.name());
- };
- };
-
-
- /**
- * Add field to collection
- */
- public void add (String key, MetaField mf) {
- fieldsMap.put(key, mf);
+ public MetaFields (String id) {
+ this.addMessage(0, "Response format is temporary");
};
/**
- * Get field from collection
- */
- public MetaField get (String key) {
- return fieldsMap.get(key);
- };
-
-
- /**
* Serialize response as a {@link JsonNode}.
*
* @return {@link JsonNode} representation of the response
@@ -201,12 +42,11 @@
doc.put("@type", "koral:document");
ArrayNode fields = doc.putArray("fields");
-
// Iterate over all fields
- Iterator fIter = fieldsMap.keySet().iterator();
+ Iterator<MetaField> fIter = mFields.iterator();
while (fIter.hasNext()) {
- MetaField mf = fieldsMap.get(fIter.next());
+ MetaField mf = fIter.next();
fields.add(mf.toJsonNode());
};
diff --git a/src/main/java/de/ids_mannheim/korap/response/MetaFieldsExt.java b/src/main/java/de/ids_mannheim/korap/response/MetaFieldsObj.java
similarity index 97%
rename from src/main/java/de/ids_mannheim/korap/response/MetaFieldsExt.java
rename to src/main/java/de/ids_mannheim/korap/response/MetaFieldsObj.java
index 460ae15..82534ba 100644
--- a/src/main/java/de/ids_mannheim/korap/response/MetaFieldsExt.java
+++ b/src/main/java/de/ids_mannheim/korap/response/MetaFieldsObj.java
@@ -26,7 +26,7 @@
import org.apache.lucene.index.*;
-public class MetaFieldsExt implements Iterable<MetaField> {
+public class MetaFieldsObj implements Iterable<MetaField> {
// Logger
private final static Logger log = LoggerFactory.getLogger(MetaFields.class);
@@ -45,7 +45,7 @@
private Map<String, MetaField> fieldsMap = new HashMap<>();
- public MetaFieldsExt () {};
+ public MetaFieldsObj () {};
/**
@@ -112,6 +112,7 @@
// Field is a number
else {
+ mf.type = "type:integer";
mf.values.add(n.toString());
};
}
diff --git a/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java b/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
index 01f3c86..8518f90 100644
--- a/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
+++ b/src/test/java/de/ids_mannheim/korap/index/TestKrillIndex.java
@@ -198,8 +198,11 @@
fd.addStored("ref", "My reference");
fd.addAttachement("ref2", "data:text/plain;charset=UTF-8,My reference2");
+/*
fd.addKeywords("keyword", "baum");
fd.addKeywords("keyword", "wald");
+*/
+ fd.addKeywords("keyword", "baum wald");
fd.addText("title", "Der Name der Rose");