Support fields parameter in getFields() and fields order in metadata responses (fixes #46)
Change-Id: Ie479a60bd247c1989c89dcffb5383271a8ba4a15
diff --git a/src/main/java/de/ids_mannheim/korap/Krill.java b/src/main/java/de/ids_mannheim/korap/Krill.java
index d395fec..6bee230 100644
--- a/src/main/java/de/ids_mannheim/korap/Krill.java
+++ b/src/main/java/de/ids_mannheim/korap/Krill.java
@@ -61,10 +61,6 @@
private final ObjectMapper mapper = new ObjectMapper();
- // Logger
- private final static Logger log = LoggerFactory.getLogger(Krill.class);
-
-
/**
* Construct a new Krill object.
*/
@@ -305,7 +301,6 @@
// Apply search
else {
-
// This contains meta and matches
kr = this.index.search(this);
// this.getCollection().setIndex(this.index);
@@ -313,6 +308,7 @@
};
kr.setQuery(this.getQuery());
+
kr.setCollection(this.getCollection());
kr.setMeta(this.getMeta());
diff --git a/src/main/java/de/ids_mannheim/korap/KrillIndex.java b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
index 6a7e495..a365045 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
@@ -4,13 +4,7 @@
import java.io.InputStream;
import java.nio.ByteBuffer;
// Java core classes
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Properties;
+import java.util.*;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
@@ -818,7 +812,7 @@
uid = new Integer(Integer.parseInt(uid)).toString();
Filter filter = (Filter) new QueryWrapperFilter(
- new TermQuery(new Term("UID", uid)));
+ new TermQuery(new Term("UID", uid)));
try {
@@ -1110,24 +1104,28 @@
field);
// The following fields should be lifted for the match
- HashSet<String> fields = (HashSet<String>) new Krill().getMeta()
- .getFields().clone();
-
- // Lift primary field
- fields.add(field);
+ List<String> fields = (ArrayList<String>) new Krill().getMeta()
+ .getFields().clone();
// Lift all fields
if (fields.contains("@all"))
fields = null;
+ HashSet<String> fieldsSet = new HashSet<String>(fields);
+
+ // Lift primary field
+ fieldsSet.add(field);
+
// Load the necessary fields of the document
- Document doc = atomic.reader().document(localDocID, fields);
+ Document doc = (fields != null)
+ ? atomic.reader().document(localDocID, fieldsSet)
+ : atomic.reader().document(localDocID);
// Put some more information to the match
PositionsToOffset pto = new PositionsToOffset(atomic, field);
match.setPositionsToOffset(pto);
match.setLocalDocID(localDocID);
- match.populateDocument(doc, field, fields);
+ match.populateDocument(doc, field, (List<String>) fields);
if (DEBUG)
log.trace("The document has the id '{}' or the sigle '{}'",
match.getDocID(), match.getTextSigle());
@@ -1346,15 +1344,18 @@
kr.setVersion(this.getVersion());
// The following fields should be lifted for matches
- HashSet<String> fields = (HashSet<String>) meta.getFields().clone();
-
- // Lift primary field
- fields.add(field);
+ List<String> fields = (ArrayList<String>) meta.getFields().clone();
+ HashSet<String> fieldsSet = new HashSet<String>(fields);
// Lift all fields
- if (fields.contains("@all"))
+ if (fields.contains("@all")) {
fields = null;
-
+ }
+ else {
+ // Lift primary field
+ fieldsSet.add(field);
+ };
+
// Some initializations ...
int i = 0;
int startIndex = kr.getStartIndex();
@@ -1479,7 +1480,7 @@
// Do not load all of this, in case the doc is the same!
final Document doc = (fields != null)
- ? lreader.document(localDocID, fields)
+ ? lreader.document(localDocID, fieldsSet)
: lreader.document(localDocID);
// Create new Match
@@ -1593,20 +1594,24 @@
return kr;
};
+ public MetaFields getFields (String textSigle) {
+
+ List hs = new ArrayList<String>();
+ hs.add("@all");
+ return this.getFields(textSigle, hs);
+ };
+
// Return field values
- public MetaFields getFields (String textSigle) {
- // , HashSet<String> fields) {
+ public MetaFields getFields (String textSigle, List<String> fields) {
// Create TermQuery for document
TermQuery textSigleQuery = new TermQuery(new Term("textSigle", textSigle));
Filter filter = (Filter) new QueryWrapperFilter(textSigleQuery);
- /*
- if (fields.contain("@all"))
+ if (fields.contains("@all"))
fields = null;
- */
MetaFields metaFields = new MetaFields(textSigle);
@@ -1632,7 +1637,10 @@
continue;
Document doc = atomic.reader().document(localDocID);
- metaFields.populateFields(doc);
+ if (fields == null)
+ metaFields.populateFields(doc);
+ else
+ metaFields.populateFields(doc, fields);
return metaFields;
};
diff --git a/src/main/java/de/ids_mannheim/korap/KrillMeta.java b/src/main/java/de/ids_mannheim/korap/KrillMeta.java
index 330cbdb..a3db457 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillMeta.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillMeta.java
@@ -26,7 +26,7 @@
private short itemsPerResource = 0;
private SearchContext context;
- private HashSet<String> fields;
+ private ArrayList<String> fields;
HashSet<Integer> highlights;
// Timeout search after milliseconds
@@ -37,9 +37,7 @@
private final static Logger log = LoggerFactory.getLogger(Krill.class);
{
- fields = new HashSet<String>(16);
-
- // TODO: Support @all
+ fields = new ArrayList<String>(16);
// Lift following fields per default
// These fields are chosen for <legacy /> reasons
@@ -171,8 +169,9 @@
this.addField(field.asText());
};
}
- else
+ else {
this.addField(json.get("fields").asText());
+ };
};
return this;
@@ -265,7 +264,7 @@
/**
* Get the fields as a set
*/
- public HashSet<String> getFields () {
+ public ArrayList<String> getFields () {
return this.fields;
};
diff --git a/src/main/java/de/ids_mannheim/korap/index/AbstractDocument.java b/src/main/java/de/ids_mannheim/korap/index/AbstractDocument.java
index 8856d98..8de0f35 100644
--- a/src/main/java/de/ids_mannheim/korap/index/AbstractDocument.java
+++ b/src/main/java/de/ids_mannheim/korap/index/AbstractDocument.java
@@ -115,7 +115,7 @@
* Primary data field.
*/
public void populateDocument (Document doc, String field) {
- HashSet<String> fieldList = new HashSet<>(32);
+ List<String> fieldList = new ArrayList<>(32);
Iterator<IndexableField> fieldIterator = doc.getFields().iterator();
while (fieldIterator.hasNext())
fieldList.add(fieldIterator.next().name());
@@ -135,23 +135,26 @@
* Hash object with all supported fields.
*/
public void populateDocument (Document doc, String field,
- Collection<String> fields) {
+ List<String> fields) {
this.setPrimaryData(doc.get(field));
this.populateFields(doc, fields);
};
public void populateFields (Document doc) {
- HashSet<String> fieldList = new HashSet<>(32);
+ ArrayList<String> fieldList = new ArrayList<>(32);
Iterator<IndexableField> fieldIterator = doc.getFields().iterator();
- while (fieldIterator.hasNext())
+ while (fieldIterator.hasNext()) {
fieldList.add(fieldIterator.next().name());
+ };
+ // TODO: Sort alphabetically!
+
this.populateFields(doc, fieldList);
};
- public void populateFields (Document doc, Collection<String> fields) {
+ public void populateFields (Document doc, List<String> fields) {
// Remove all fields already set
Iterator<String> fieldsIter = fields.iterator();
while (fieldsIter.hasNext()) {
@@ -160,10 +163,13 @@
};
};
- if (fields.contains("UID"))
+
+ if (fields.contains("UID")) {
this.setUID(doc.get("UID"));
+ };
fieldsIter = fields.iterator();
+ mFields.fieldsOrder = new ArrayList<>(16);
while (fieldsIter.hasNext()) {
String name = fieldsIter.next();
@@ -172,11 +178,12 @@
if (name == "tokens" || name == "UID")
continue;
+ mFields.fieldsOrder.add(name);
+
IndexableField iField = doc.getField(name);
if (iField == null)
continue;
-
MetaField mf = mFields.add(iField);
@@ -352,17 +359,21 @@
HashMap<String, JsonNode> map = new HashMap<>();
while (mfIterator.hasNext()) {
- String mfs = mfIterator.next().key;
- if (legacyDateFields.contains(mfs) ||
- legacyStoredFields.contains(mfs) ||
- legacyTextFields.contains(mfs) ||
- legacyStringFields.contains(mfs) ||
- legacyKeywordsFields.contains(mfs)
+ MetaField mf = mfIterator.next();
+ if (mf == null)
+ continue;
+ String mfs = mf.key;
+ String value = this.getFieldValue(mfs);
+ if (value != null && (
+ legacyDateFields.contains(mfs) ||
+ legacyStoredFields.contains(mfs) ||
+ legacyTextFields.contains(mfs) ||
+ legacyStringFields.contains(mfs) ||
+ legacyKeywordsFields.contains(mfs) ||
+ legacyDateFields.contains(mfs)
+ )
) {
- map.put(mfs, new TextNode(this.getFieldValue(mfs)));
- }
- else if (legacyDateFields.contains(mfs)) {
- map.put(mfs, new TextNode(this.getFieldValue(mfs)));
+ map.put(mfs, new TextNode(value));
}
};
diff --git a/src/main/java/de/ids_mannheim/korap/response/MetaField.java b/src/main/java/de/ids_mannheim/korap/response/MetaField.java
index 94052d7..e6b330e 100644
--- a/src/main/java/de/ids_mannheim/korap/response/MetaField.java
+++ b/src/main/java/de/ids_mannheim/korap/response/MetaField.java
@@ -17,11 +17,10 @@
// Mapper for JSON serialization
ObjectMapper mapper = new ObjectMapper();
- public String type = "type:string";
+ public String type;
public String key;
public List<String> values = new ArrayList<>();
-
public MetaField (String key) {
this.key = key;
};
@@ -52,9 +51,13 @@
public JsonNode toJsonNode () {
ObjectNode json = mapper.createObjectNode();
json.put("@type", "koral:field");
- json.put("type", this.type);
json.put("key", this.key);
+ if (this.type == null)
+ return json;
+
+ json.put("type", this.type);
+
// Value is numerical
if (this.type.equals("type:integer")) {
diff --git a/src/main/java/de/ids_mannheim/korap/response/MetaFieldsObj.java b/src/main/java/de/ids_mannheim/korap/response/MetaFieldsObj.java
index ea992b2..c6ba70e 100644
--- a/src/main/java/de/ids_mannheim/korap/response/MetaFieldsObj.java
+++ b/src/main/java/de/ids_mannheim/korap/response/MetaFieldsObj.java
@@ -40,6 +40,8 @@
// Mapper for JSON serialization
ObjectMapper mapper = new ObjectMapper();
+ public List<String> fieldsOrder;
+
private Map<String, MetaField> fieldsMap = new HashMap<>();
@@ -53,7 +55,7 @@
return this.add(
metaFieldFromIndexableField(
iField,
- new MetaField(iField.name())
+ new MetaField(iField.name(), "type:string")
)
);
};
@@ -76,7 +78,7 @@
return mf;
};
-
+
// Field type needs to be restored heuristically
// - though that's not very elegant
public static MetaField metaFieldFromIndexableField (IndexableField iField, MetaField mf) {
@@ -206,14 +208,20 @@
return fieldsMap.containsKey(key);
};
+ private Iterator<String> getIterator () {
+ if (this.fieldsOrder == null) {
+ return fieldsMap.keySet().iterator();
+ };
+ return this.fieldsOrder.iterator();
+ };
@Override
public Iterator<MetaField> iterator() {
return new Iterator<MetaField>() {
- private Iterator it = fieldsMap.keySet().iterator();
-
+ private Iterator<String> it = getIterator();
+
private int currentIndex = 0;
@Override
@@ -223,7 +231,11 @@
@Override
public MetaField next() {
- return fieldsMap.get(it.next());
+ String key = it.next();
+ MetaField mf = fieldsMap.get(key);
+ if (mf == null)
+ return new MetaField(key);
+ return mf;
};
@Override