collection rewrite section fix
diff --git a/src/main/java/de/ids_mannheim/korap/resource/rewrite/CollectionCleanupFilter.java b/src/main/java/de/ids_mannheim/korap/resource/rewrite/CollectionCleanupFilter.java
index 3f9ce89..3985bd0 100644
--- a/src/main/java/de/ids_mannheim/korap/resource/rewrite/CollectionCleanupFilter.java
+++ b/src/main/java/de/ids_mannheim/korap/resource/rewrite/CollectionCleanupFilter.java
@@ -7,6 +7,7 @@
import de.ids_mannheim.korap.config.KustvaktConfiguration;
import de.ids_mannheim.korap.user.User;
import de.ids_mannheim.korap.utils.JsonUtils;
+import edu.emory.mathcs.backport.java.util.Arrays;
import java.util.Iterator;
@@ -19,7 +20,7 @@
@Override
public JsonNode preProcess (KoralNode node, KustvaktConfiguration config,
User user) {
- return node.rawNode();
+ return process(node.rawNode());
}
@@ -49,8 +50,10 @@
if (!root.equals(sub)) {
if (sub.isObject()) {
ObjectNode ob = (ObjectNode) root;
- ob.removeAll();
+ ob.remove(Arrays.asList(new String[]{"@type", "operation", "operands"}));
ob.putAll((ObjectNode) sub);
+ //if (!rewrites.isMissingNode())
+ // ob.put("rewrites", rewrites);
}
}
}
@@ -59,7 +62,7 @@
@Override
public JsonNode postProcess (KoralNode node) {
- return process(node.rawNode());
+ return null;
}
diff --git a/src/main/java/de/ids_mannheim/korap/resource/rewrite/CollectionConstraint.java b/src/main/java/de/ids_mannheim/korap/resource/rewrite/CollectionConstraint.java
index 05b6168..f725c9a 100644
--- a/src/main/java/de/ids_mannheim/korap/resource/rewrite/CollectionConstraint.java
+++ b/src/main/java/de/ids_mannheim/korap/resource/rewrite/CollectionConstraint.java
@@ -24,7 +24,7 @@
if (node.get("@type").equals("koral:doc")) {
if (node.get("key").equals(Attributes.CORPUS_SIGLE)
&& !check(node, user))
- node.removeNode();
+ node.removeNode(Attributes.CORPUS_SIGLE);
}
return node.rawNode();
}
diff --git a/src/main/java/de/ids_mannheim/korap/resource/rewrite/KoralNode.java b/src/main/java/de/ids_mannheim/korap/resource/rewrite/KoralNode.java
index 8df544e..dbce70e 100644
--- a/src/main/java/de/ids_mannheim/korap/resource/rewrite/KoralNode.java
+++ b/src/main/java/de/ids_mannheim/korap/resource/rewrite/KoralNode.java
@@ -29,10 +29,12 @@
}
- public void buildRewrites() {
- System.out.println("LIST REWRITES "+ this.rewrites.rewrites);
- this.rewrites.build(this.node);
+ public void buildRewrites(JsonNode node) {
+ this.rewrites.build(node);
+ }
+ public void buildRewrites() {
+ this.rewrites.build(this.node);
}
@Override
@@ -129,8 +131,8 @@
}
- public void removeNode () {
- this.rewrites.add("deletion", this.node);
+ public void removeNode (String identifier) {
+ this.rewrites.add("deletion", identifier);
this.remove = true;
}
diff --git a/src/main/java/de/ids_mannheim/korap/resource/rewrite/RewriteHandler.java b/src/main/java/de/ids_mannheim/korap/resource/rewrite/RewriteHandler.java
index fb98864..ef0c1d2 100644
--- a/src/main/java/de/ids_mannheim/korap/resource/rewrite/RewriteHandler.java
+++ b/src/main/java/de/ids_mannheim/korap/resource/rewrite/RewriteHandler.java
@@ -114,64 +114,13 @@
}
- public void clear () {
- this.node_processors.clear();
- this.query_processors.clear();
- this.token_node_processors.clear();
- }
-
-
- private boolean processNode (String name, JsonNode root, User user, boolean post) {
- if (root.isObject()) {
- if (root.has("operands")) {
- JsonNode ops = root.at("/operands");
- Iterator<JsonNode> it = ops.elements();
- while (it.hasNext()) {
- JsonNode next = it.next();
- if (processNode(name, next, user, post))
- it.remove();
- }
- }
- else if (root.path("@type").asText().equals("koral:token")) {
- // todo: koral:token nodes cannot be flagged for deletion --> creates the possibility for empty koral:token nodes
- rewrite(name, KoralNode.wrapNode(root), user,
- this.token_node_processors, post);
- return processNode(name, root.path("wrap"), user, post);
- }
- else {
- return rewrite(name, KoralNode.wrapNode(root), user,
- this.node_processors, post);
- }
- }
- else if (root.isArray()) {
- Iterator<JsonNode> it = root.elements();
- while (it.hasNext()) {
- JsonNode next = it.next();
- if (processNode(name, next, user, post))
- it.remove();
- }
- }
- return false;
- }
-
-
- private JsonNode iterate (JsonNode root, User user, boolean post) {
- jlog.debug("Running rewrite process on query {}", root);
- if (root != null) {
- Iterator<Map.Entry<String, JsonNode>> it = root.fields();
- while (it.hasNext()) {
- Map.Entry<String, JsonNode> next = it.next();
- processNode(next.getKey(), next.getValue(), user, post);
- }
- processFixedNode(root, user, this.query_processors, post);
- }
- return root;
- }
public JsonNode process(JsonNode root, User user) {
- JsonNode pre = iterate(root, user, false);
- return iterate(pre, user, true);
+ RewriteProcess process = new RewriteProcess(root, user);
+ JsonNode pre = process.start(false);
+ //return iterate(pre, user, true);
+ return pre;
}
@@ -180,78 +129,10 @@
}
- /**
- * @param node
- * @param user
- * @param tasks
- * @return boolean true if node is to be removed from parent! Only
- * applies if parent is an array node
- */
- private boolean rewrite (String rootNode, KoralNode node, User user,
- Collection<? extends RewriteTask> tasks, boolean post) {
- if (this.config == null)
- throw new RuntimeException("KustvaktConfiguration must be set!");
-
- for (RewriteTask task : tasks) {
- jlog.debug("running processor on node: " + node);
- jlog.debug("on processor: " + task.getClass().toString());
-
- if (this.beans != null && task instanceof BeanInjectable)
- ((BeanInjectable) task).insertBeans(this.beans);
-
- if (task instanceof RewriteTask.IterableRewritePath) {
- RewriteTask.IterableRewritePath rw = (RewriteTask.IterableRewritePath) task;
- if (rw.path() != null && !rw.path().equals(rootNode)) {
- jlog.debug("skipping node: " + node);
- continue;
- }
- }
- try {
- if (!post && task instanceof RewriteTask.RewriteBefore) {
- ((RewriteTask.RewriteBefore) task).preProcess(node,
- this.config, user);
- }
- else if (task instanceof RewriteTask.RewriteAfter) {
- ((RewriteTask.RewriteAfter) task).postProcess(node);
- }
- node.buildRewrites();
- }
- catch (KustvaktException e) {
- jlog.error("Error in rewrite processor {} for node {}", task
- .getClass().getSimpleName(), node.rawNode().toString());
- e.printStackTrace();
- }
- if (node.isRemove())
- break;
- }
- return node.isRemove();
- }
-
-
- private void processFixedNode (JsonNode node, User user,
- Collection<RewriteTask> tasks, boolean post) {
- for (RewriteTask task : tasks) {
- KoralNode next = KoralNode.wrapNode(node);
- if (task instanceof RewriteTask.RewriteNodeAt) {
- RewriteTask.RewriteNodeAt rwa = (RewriteTask.RewriteNodeAt) task;
- if ((rwa.at() != null && !node.at(rwa.at()).isMissingNode()))
- next = next.at(rwa.at());
- }
-
- try {
- if (!post & task instanceof RewriteTask.RewriteBefore)
- ((RewriteTask.RewriteBefore) task).preProcess(next,
- this.config, user);
- else if (task instanceof RewriteTask.RewriteAfter)
- ((RewriteTask.RewriteAfter) task).postProcess(next);
- next.buildRewrites();
- }
- catch (KustvaktException e) {
- jlog.error("Error in rewrite processor {} for node {}", task
- .getClass().getSimpleName(), next.toString());
- e.printStackTrace();
- }
- }
+ public void clear () {
+ this.node_processors.clear();
+ this.query_processors.clear();
+ this.token_node_processors.clear();
}
@@ -260,4 +141,144 @@
this.beans = beans;
this.config = beans.getConfiguration();
}
+
+
+
+ public class RewriteProcess {
+
+ private JsonNode root;
+ private User user;
+
+ private RewriteProcess(JsonNode root, User user) {
+ this.root = root;
+ this.user = user;
+ }
+
+ private KoralNode processNode (String key, JsonNode value, boolean post) {
+ KoralNode kroot = KoralNode.wrapNode(value);
+ if (value.isObject()) {
+ if (value.has("operands")) {
+ JsonNode ops = value.at("/operands");
+ Iterator<JsonNode> it = ops.elements();
+ while (it.hasNext()) {
+ JsonNode next = it.next();
+ KoralNode kn = processNode(key, next, post);
+ if (kn.isRemove())
+ it.remove();
+ }
+ }
+ else if (value.path("@type").asText().equals("koral:token")) {
+ // todo: koral:token nodes cannot be flagged for deletion --> creates the possibility for empty koral:token nodes
+ rewrite(key, kroot,
+ RewriteHandler.this.token_node_processors, post);
+ return processNode(key, value.path("wrap"), post);
+ }
+ else {
+ return rewrite(key, kroot,
+ RewriteHandler.this.node_processors, post);
+ }
+ }
+ else if (value.isArray()) {
+ Iterator<JsonNode> it = value.elements();
+ while (it.hasNext()) {
+ JsonNode next = it.next();
+ KoralNode kn = processNode(key, next, post);
+ if (kn.isRemove())
+ it.remove();
+ }
+ }
+ return kroot;
+ }
+
+
+ private JsonNode start(boolean post) {
+ jlog.debug("Running rewrite process on query {}", root);
+ if (root != null) {
+ Iterator<Map.Entry<String, JsonNode>> it = root.fields();
+ while (it.hasNext()) {
+ Map.Entry<String, JsonNode> next = it.next();
+ processNode(next.getKey(), next.getValue(), post);
+ }
+ processFixedNode(root, RewriteHandler.this.query_processors, post);
+ }
+ return root;
+ }
+
+ /**
+ * @param node
+ * @param tasks
+ * @return boolean true if node is to be removed from parent! Only
+ * applies if parent is an array node
+ */
+ private KoralNode rewrite (String rootNode, KoralNode node,
+ Collection<? extends RewriteTask> tasks, boolean post) {
+ if (RewriteHandler.this.config == null)
+ throw new RuntimeException("KustvaktConfiguration must be set!");
+
+ for (RewriteTask task : tasks) {
+ jlog.debug("running processor on node: " + node);
+ jlog.debug("on processor: " + task.getClass().toString());
+
+ if (RewriteHandler.this.beans != null && task instanceof BeanInjectable)
+ ((BeanInjectable) task).insertBeans(RewriteHandler.this.beans);
+
+ if (task instanceof RewriteTask.IterableRewritePath) {
+ RewriteTask.IterableRewritePath rw = (RewriteTask.IterableRewritePath) task;
+ if (rw.path() != null && !rw.path().equals(rootNode)) {
+ jlog.debug("skipping node: " + node);
+ continue;
+ }
+ }
+ try {
+ if (!post && task instanceof RewriteTask.RewriteBefore) {
+ ((RewriteTask.RewriteBefore) task).preProcess(node,
+ RewriteHandler.this.config, this.user);
+ }
+ else if (task instanceof RewriteTask.RewriteAfter) {
+ ((RewriteTask.RewriteAfter) task).postProcess(node);
+ }
+ }
+ catch (KustvaktException e) {
+ jlog.error("Error in rewrite processor {} for node {}", task
+ .getClass().getSimpleName(), node.rawNode().toString());
+ e.printStackTrace();
+ }
+ if (node.isRemove()) {
+ node.buildRewrites(this.root.at("/"+ rootNode));
+ break;
+ } else
+ node.buildRewrites();
+ }
+ return node;
+ }
+
+ // fixme: merge with processNode!
+ private void processFixedNode (JsonNode node,
+ Collection<RewriteTask> tasks, boolean post) {
+ for (RewriteTask task : tasks) {
+ KoralNode next = KoralNode.wrapNode(node);
+ if (task instanceof RewriteTask.RewriteNodeAt) {
+ RewriteTask.RewriteNodeAt rwa = (RewriteTask.RewriteNodeAt) task;
+ if ((rwa.at() != null && !node.at(rwa.at()).isMissingNode()))
+ next = next.at(rwa.at());
+ }
+
+ try {
+ if (!post & task instanceof RewriteTask.RewriteBefore)
+ ((RewriteTask.RewriteBefore) task).preProcess(next,
+ RewriteHandler.this.config, user);
+ else if (task instanceof RewriteTask.RewriteAfter)
+ ((RewriteTask.RewriteAfter) task).postProcess(next);
+ next.buildRewrites();
+ }
+ catch (KustvaktException e) {
+ jlog.error("Error in rewrite processor {} for node {}", task
+ .getClass().getSimpleName(), next.toString());
+ e.printStackTrace();
+ }
+ }
+ }
+
+
+ }
}
diff --git a/src/test/java/de/ids_mannheim/korap/resource/rewrite/CollectionRewriteTest.java b/src/test/java/de/ids_mannheim/korap/resource/rewrite/CollectionRewriteTest.java
index fbf88c0..d655713 100644
--- a/src/test/java/de/ids_mannheim/korap/resource/rewrite/CollectionRewriteTest.java
+++ b/src/test/java/de/ids_mannheim/korap/resource/rewrite/CollectionRewriteTest.java
@@ -79,11 +79,16 @@
s.setQuery(simple_add_query, "poliqarp");
s.setCollection("corpusSigle=BRZ13 & corpusSigle=WPD");
String result = s.toJSON();
- JsonNode node = JsonUtils.readTree(handler.process(result,
- User.UserFactory.getUser("test_user")));
- // System.out.println("RESULTING REWR NODE " + node);
- assertNotNull(node);
- assertEquals(0, node.at("/collection/operands").size());
+ try {
+ JsonNode node = JsonUtils.readTree(handler.process(result,
+ User.UserFactory.getUser("test_user")));
+ System.out.println(node);
+ assertNotNull(node);
+ assertEquals(0, node.at("/collection/operands").size());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
+ } catch ( Exception e) {
+ e.printStackTrace();
+ }
}
@@ -104,6 +109,7 @@
.asText());
assertEquals("textClass",
node.at("/collection/operands/0/operands/0/key").asText());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
}
@@ -131,6 +137,7 @@
.asText());
assertEquals("textClass", node.at("/collection/operands/1/key")
.asText());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
}
@@ -142,12 +149,15 @@
handler.add(CollectionCleanupFilter.class);
QuerySerializer s = new QuerySerializer();
s.setQuery(simple_add_query, "poliqarp");
- s.setCollection("(corpusSigle=BRZ13 & textClass=Wissenschaft)");
+ s.setCollection("(corpusSigle=BRZ13 & textClass=wissenschaft)");
String result = s.toJSON();
JsonNode node = JsonUtils.readTree(handler.process(result,
User.UserFactory.getUser("test_user")));
assertNotNull(node);
assertEquals("koral:doc", node.at("/collection/@type").asText());
+ assertEquals("textClass", node.at("/collection/key").asText());
+ assertEquals("wissenschaft", node.at("/collection/value").asText());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
}
@@ -167,6 +177,7 @@
assertNotNull(node);
assertEquals("koral:doc", node.at("/collection/@type").asText());
assertEquals("textClass", node.at("/collection/key").asText());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
}
@@ -188,6 +199,9 @@
assertEquals(2, node.at("/collection/operands").size());
assertEquals("koral:doc", node.at("/collection/operands/0/@type")
.asText());
+ assertEquals("koral:doc", node.at("/collection/operands/1/@type")
+ .asText());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
}
@@ -205,7 +219,7 @@
assertNotNull(node);
assertEquals("corpusSigle", node.at("/collection/operands/0/key").asText());
assertEquals("corpusSigle", node.at("/collection/operands/1/key").asText());
- assertEquals("koral:rewrite", node.at("/collection/rewrites/@type").asText());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
}
@@ -216,7 +230,6 @@
handler.insertBeans(helper().getContext());
handler.add(PublicCollection.class);
- try {
QuerySerializer s = new QuerySerializer();
s.setQuery(simple_add_query, "poliqarp");
s.setCollection("(docSigle=WPD_AAA & textClass=wissenschaft)");
@@ -227,12 +240,58 @@
assertEquals(2, node.at("/collection/operands").size());
assertEquals("corpusSigle", node.at("/collection/operands/1/operands/0/key").asText());
assertEquals("corpusSigle", node.at("/collection/operands/1/operands/1/key").asText());
- assertEquals("koral:rewrite", node.at("/collection/rewrites/@type").asText());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
+ }
- }
- catch (Exception e) {
- e.printStackTrace();
- }
+
+ @Test
+ public void testRemoveCorpusFromDifferentGroups() {
+ RewriteHandler handler = new RewriteHandler();
+ handler.insertBeans(helper().getContext());
+ handler.add(CollectionConstraint.class);
+ QuerySerializer s = new QuerySerializer();
+ s.setQuery(simple_add_query, "poliqarp");
+ s.setCollection("(corpusSigle=BRZ14 & textClass=wissenschaft) | (corpusSigle=AZPR | textClass=freizeit)");
+ String org = s.toJSON();
+ JsonNode node = JsonUtils.readTree(handler.process(org,
+ User.UserFactory.getUser("test_user")));
+ assertNotNull(node);
+ assertEquals("koral:docGroup", node.at("/collection/@type").asText());
+ assertEquals(2, node.at("/collection/operands").size());
+ assertEquals("koral:docGroup", node.at("/collection/operands/0/@type")
+ .asText());
+ assertEquals("koral:docGroup", node.at("/collection/operands/1/@type")
+ .asText());
+ assertEquals(1, node.at("/collection/operands/0/operands").size());
+ assertEquals(1, node.at("/collection/operands/1/operands").size());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
+ }
+
+ @Test
+ public void testRemoveOneCorpusAndMoveDocFromTwoGroups() {
+ RewriteHandler handler = new RewriteHandler();
+ handler.insertBeans(helper().getContext());
+ handler.add(CollectionConstraint.class);
+ // todo: use this collection query also to test clean up filter! after reduction of corpusSigle (wiss | freizeit)!
+ handler.add(CollectionCleanupFilter.class);
+ QuerySerializer s = new QuerySerializer();
+ s.setQuery(simple_add_query, "poliqarp");
+ s.setCollection("(corpusSigle=BRZ14 & textClass=wissenschaft) | (corpusSigle=AZPR | textClass=freizeit)");
+ String org = s.toJSON();
+ JsonNode node = JsonUtils.readTree(handler.process(org,
+ User.UserFactory.getUser("test_user")));
+ assertNotNull(node);
+ assertEquals("koral:docGroup", node.at("/collection/@type").asText());
+ assertEquals(2, node.at("/collection/operands").size());
+ assertEquals("koral:doc", node.at("/collection/operands/0/@type")
+ .asText());
+ assertEquals("koral:doc", node.at("/collection/operands/0/@type").asText());
+ assertEquals("textClass", node.at("/collection/operands/0/key").asText());
+ assertEquals("wissenschaft", node.at("/collection/operands/0/value").asText());
+ assertEquals("koral:doc", node.at("/collection/operands/1/@type").asText());
+ assertEquals("textClass", node.at("/collection/operands/1/key").asText());
+ assertEquals("freizeit", node.at("/collection/operands/1/value").asText());
+ assertEquals("koral:rewrite", node.at("/collection/rewrites/0/@type").asText());
}