Implemented pipe extension in the search API.
Change-Id: If2a486185a7d16a27b7b46d35d85b7d5f27b66cd
diff --git a/core/Changes b/core/Changes
index 272c8d1..64c0651 100644
--- a/core/Changes
+++ b/core/Changes
@@ -1,3 +1,7 @@
+# version 0.62.3
+03/12/2019
+ - Implemented pipe extension in the search API (margaretha)
+
# version 0.62.2
13/11/2019
- Added warnings when requesting non-public fields via the search API with
diff --git a/core/pom.xml b/core/pom.xml
index 7bef3cd..be3b949 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-core</artifactId>
- <version>0.62.2</version>
+ <version>0.62.3</version>
<properties>
<java.version>1.8</java.version>
@@ -210,7 +210,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.7.25</version>
+ <version>1.7.29</version>
</dependency>
<!-- Java Assist -->
diff --git a/core/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java b/core/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
index 8144124..dcbc3cb 100644
--- a/core/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
+++ b/core/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
@@ -1,7 +1,11 @@
package de.ids_mannheim.korap.config;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -12,6 +16,8 @@
import java.util.Set;
import java.util.regex.Pattern;
+import org.apache.commons.io.FileUtils;
+
import de.ids_mannheim.korap.util.KrillProperties;
import de.ids_mannheim.korap.utils.TimeUtils;
import lombok.Getter;
@@ -100,8 +106,11 @@
// another variable might be needed to define which metadata fields are restricted
private boolean isMetadataRestricted = false;
+ public static Map<String, String> pipes = new HashMap<>();
+
public KustvaktConfiguration (Properties properties) throws Exception {
load(properties);
+ readPipesFile("pipes");
KrillProperties.setProp(properties);
}
@@ -191,6 +200,27 @@
// properties.getProperty("security.passcode.salt",
// "accountCreation");
}
+
+ public void readPipesFile (String filename) throws IOException {
+ File file = new File(filename);
+ if (file.exists()) {
+ BufferedReader br = new BufferedReader(
+ new InputStreamReader(new FileInputStream(file)));
+
+ String line = null;
+ while( (line=br.readLine())!=null ){
+ String[] parts = line.split("\t");
+ if (parts.length !=2){
+ continue;
+ }
+ else{
+ pipes.put(parts[0], parts[1]);
+ }
+ }
+ br.close();
+ }
+ }
+
/**
* set properties
diff --git a/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java b/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
index 3a04452..46452ca 100644
--- a/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
+++ b/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
@@ -8,6 +8,7 @@
import javax.annotation.PostConstruct;
import javax.servlet.ServletContext;
import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.UriBuilder;
@@ -17,6 +18,9 @@
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.JsonNode;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import de.ids_mannheim.de.init.VCLoader;
@@ -120,7 +124,7 @@
@SuppressWarnings("unchecked")
public String search (String engine, String username, HttpHeaders headers,
String q, String ql, String v, String cq, String fields,
- Integer pageIndex, Integer pageInteger, String ctx,
+ String pipes, Integer pageIndex, Integer pageInteger, String ctx,
Integer pageLength, Boolean cutoff, boolean accessRewriteDisabled)
throws KustvaktException {
@@ -129,6 +133,11 @@
"page must start from 1", "page");
}
+ String[] pipeArray = null;
+ if (pipes!=null && !pipes.isEmpty()){
+ pipeArray = pipes.split(",");
+ }
+
KustvaktConfiguration.BACKENDS eng = this.config.chooseBackend(engine);
User user = createUser(username, headers);
CorpusAccess corpusAccess = user.getCorpusAccess();
@@ -157,8 +166,10 @@
throw new KustvaktException(serializer.toJSON());
}
- String query =
- this.rewriteHandler.processQuery(serializer.toJSON(), user);
+ String query = serializer.toJSON();
+ query = runPipes(query,pipeArray);
+
+ query = this.rewriteHandler.processQuery(query, user);
if (DEBUG){
jlog.debug("the serialized query " + query);
}
@@ -175,6 +186,23 @@
}
+ private String runPipes (String query, String[] pipeArray) {
+ if (pipeArray !=null){
+ for (int i=0; i<pipeArray.length; i++){
+ String url = KustvaktConfiguration.pipes.get(pipeArray[i]);
+ // update query by sending it to a pipe URL
+ // NOTE: request formulation may vary depending on the service
+ Client client = Client.create();
+ WebResource resource = client.resource(url);
+ ClientResponse response =
+ resource.type(MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class, query);
+ query = response.getEntity(String.class);
+ }
+ }
+ return query;
+ }
+
private void handleNonPublicFields (List<String> fieldList,
boolean accessRewriteDisabled, QuerySerializer serializer) {
List<String> nonPublicFields = new ArrayList<>();
diff --git a/core/src/main/java/de/ids_mannheim/korap/test/TestController.java b/core/src/main/java/de/ids_mannheim/korap/test/TestController.java
new file mode 100644
index 0000000..43144bb
--- /dev/null
+++ b/core/src/main/java/de/ids_mannheim/korap/test/TestController.java
@@ -0,0 +1,33 @@
+package de.ids_mannheim.korap.test;
+
+import java.io.InputStream;
+
+import javax.ws.rs.Produces;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import org.springframework.stereotype.Controller;
+
+/**
+ * Controllers used only for testing
+ *
+ * @author margaretha
+ *
+ */
+@Controller
+@Path("/{version}/test")
+@Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
+public class TestController {
+
+ @POST
+ @Path("glemm")
+ @Consumes(MediaType.APPLICATION_JSON)
+ public Response dummyGlemm (String jsonld) {
+ InputStream is = getClass().getClassLoader()
+ .getResourceAsStream("test-pipes.jsonld");
+ return Response.ok(is).build();
+ }
+}
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java b/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
index 8ccb8f1..76bf89d 100644
--- a/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
+++ b/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
@@ -136,6 +136,40 @@
}
}
+ /** Performs for the given query
+ *
+ * @param securityContext
+ * @param request
+ * @param headers
+ * @param locale
+ * @param q
+ * query
+ * @param ql
+ * query language
+ * @param v
+ * query language version
+ * @param ctx
+ * result context
+ * @param cutoff
+ * determines to limit search results to one page only
+ * or not (default false)
+ * @param pageLength
+ * the number of results should be included in a page
+ * @param pageIndex
+ * @param pageInteger page number
+ * @param fields
+ * metadata fields to be included, separated by comma
+ * @param pipes
+ * external plugins for additional processing,
+ * separated by comma
+ * @param accessRewriteDisabled
+ * determine if access rewrite should be disabled
+ * (default false)
+ * @param cq
+ * corpus query defining a virtual corpus
+ * @param engine
+ * @return search results in JSON
+ */
@GET
@Path("{version}/search")
public Response searchGet (@Context SecurityContext securityContext,
@@ -148,6 +182,7 @@
@QueryParam("offset") Integer pageIndex,
@QueryParam("page") Integer pageInteger,
@QueryParam("fields") String fields,
+ @QueryParam("pipes") String pipes,
@QueryParam("access-rewrite-disabled") boolean accessRewriteDisabled,
@QueryParam("cq") String cq,
@QueryParam("engine") String engine) {
@@ -159,8 +194,9 @@
try {
scopeService.verifyScope(context, OAuth2Scope.SEARCH);
result = searchService.search(engine, context.getUsername(),
- headers, q, ql, v, cq, fields, pageIndex, pageInteger, ctx,
- pageLength, cutoff, accessRewriteDisabled);
+ headers, q, ql, v, cq, fields, pipes, pageIndex,
+ pageInteger, ctx, pageLength, cutoff,
+ accessRewriteDisabled);
}
catch (KustvaktException e) {
throw kustvaktResponseHandler.throwit(e);
diff --git a/core/src/main/resources/test-pipes.jsonld b/core/src/main/resources/test-pipes.jsonld
new file mode 100644
index 0000000..eca8cec
--- /dev/null
+++ b/core/src/main/resources/test-pipes.jsonld
@@ -0,0 +1,26 @@
+{
+ "meta": {
+ "snippets": true,
+ "timeout": 10000
+ },
+ "query": {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "match": "match:eq",
+ "key": [
+ "der",
+ "die",
+ "das"
+ ],
+ "layer": "orth",
+ "rewrites": [{
+ "@type": "koral:rewrite",
+ "src": "Glemm",
+ "operation": "operation:override",
+ "scope": "key"
+ }]
+ }
+ },
+ "@context": "http://korap.ids-mannheim.de/ns/koral/0.3/context.jsonld"
+}
diff --git a/full/Changes b/full/Changes
index 733bb9f..6ce2b2a 100644
--- a/full/Changes
+++ b/full/Changes
@@ -1,3 +1,7 @@
+# version 0.62.3
+03/12/2019
+ - Implemented pipe extension in the search API (margaretha)
+
# version 0.62.2
17/10/2019
- Handled vulnerability CVE-2019-17195. (margaretha)
diff --git a/full/pom.xml b/full/pom.xml
index 564d3be..a1a4f02 100644
--- a/full/pom.xml
+++ b/full/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-full</artifactId>
- <version>0.62.2</version>
+ <version>0.62.3</version>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -205,7 +205,7 @@
<dependency>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-core</artifactId>
- <version>[0.62.2,)</version>
+ <version>[0.62.3,)</version>
</dependency>
<!-- LDAP -->
<dependency>
diff --git a/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java b/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
index 8a2f730..cbb6977 100644
--- a/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
@@ -32,6 +32,7 @@
public static String[] classPackages =
new String[] { "de.ids_mannheim.korap.web",
+ "de.ids_mannheim.korap.test",
"com.fasterxml.jackson.jaxrs.json"};
@Override
diff --git a/full/src/test/java/de/ids_mannheim/korap/web/controller/SearchPipeTest.java b/full/src/test/java/de/ids_mannheim/korap/web/controller/SearchPipeTest.java
new file mode 100644
index 0000000..9c55d6e
--- /dev/null
+++ b/full/src/test/java/de/ids_mannheim/korap/web/controller/SearchPipeTest.java
@@ -0,0 +1,73 @@
+package de.ids_mannheim.korap.web.controller;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.sun.jersey.api.client.ClientResponse;
+
+import de.ids_mannheim.korap.config.KustvaktConfiguration;
+import de.ids_mannheim.korap.config.SpringJerseyTest;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+public class SearchPipeTest extends SpringJerseyTest {
+
+ @Autowired
+ private KustvaktConfiguration config;
+
+ private void setTestPipes () throws IOException {
+ String filename = "test-pipes";
+ File f = new File(filename);
+ if (f.exists()) {
+ f.delete();
+ }
+ f.createNewFile();
+ OutputStreamWriter writer =
+ new OutputStreamWriter(new FileOutputStream(f));
+ writer.append("glemm\t");
+ writer.append(resource().getURI().toString());
+ writer.append(API_VERSION);
+ writer.append("/test/glemm");
+ writer.flush();
+ writer.close();
+
+ config.readPipesFile(filename);
+ }
+
+ @Test
+ public void testSearchWithPipes () throws IOException, KustvaktException {
+ setTestPipes();
+ ClientResponse response = resource().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("pipes", "glemm").get(ClientResponse.class);
+
+ String entity = response.getEntity(String.class);
+
+ JsonNode node = JsonUtils.readTree(entity);
+ assertEquals(3, node.at("/query/wrap/key").size());
+
+ assertEquals(1, node.at("/collection/rewrites").size());
+ assertEquals("operation:insertion",
+ node.at("/collection/rewrites/0/operation").asText());
+ assertEquals("availability(FREE)",
+ node.at("/collection/rewrites/0/scope").asText());
+
+ node = node.at("/query/wrap/rewrites");
+ assertEquals(2, node.size());
+ assertEquals("Glemm", node.at("/0/src").asText());
+ assertEquals("operation:override", node.at("/0/operation").asText());
+ assertEquals("key", node.at("/0/scope").asText());
+
+ assertEquals("Kustvakt", node.at("/1/src").asText());
+ assertEquals("operation:injection", node.at("/1/operation").asText());
+ assertEquals("foundry", node.at("/1/scope").asText());
+ }
+}
diff --git a/lite/Changes b/lite/Changes
index 0402601..d6991e0 100644
--- a/lite/Changes
+++ b/lite/Changes
@@ -1,3 +1,7 @@
+# version 0.62.3
+03/12/2019
+ - Implemented pipe extension in the search API (margaretha)
+
# version 0.62.2
13/11/2019
- Added tests for issue #43 (margaretha)
diff --git a/lite/pom.xml b/lite/pom.xml
index c2f72f0..f06e3d1 100644
--- a/lite/pom.xml
+++ b/lite/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-lite</artifactId>
- <version>0.62.2</version>
+ <version>0.62.3</version>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -137,7 +137,7 @@
<dependency>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-core</artifactId>
- <version>[0.62.2,)</version>
+ <version>[0.62.3,)</version>
</dependency>
<!-- Jersey test framework -->
<dependency>
diff --git a/lite/src/test/java/de/ids_mannheim/korap/config/LiteJerseyTest.java b/lite/src/test/java/de/ids_mannheim/korap/config/LiteJerseyTest.java
index 99b2703..e6536f5 100644
--- a/lite/src/test/java/de/ids_mannheim/korap/config/LiteJerseyTest.java
+++ b/lite/src/test/java/de/ids_mannheim/korap/config/LiteJerseyTest.java
@@ -30,6 +30,7 @@
public static String[] classPackages =
new String[] { "de.ids_mannheim.korap.web",
+ "de.ids_mannheim.korap.test",
"com.fasterxml.jackson.jaxrs.json"};
@Override
diff --git a/lite/src/test/java/de/ids_mannheim/korap/web/service/LiteSearchPipeTest.java b/lite/src/test/java/de/ids_mannheim/korap/web/service/LiteSearchPipeTest.java
new file mode 100644
index 0000000..5e8d77f
--- /dev/null
+++ b/lite/src/test/java/de/ids_mannheim/korap/web/service/LiteSearchPipeTest.java
@@ -0,0 +1,65 @@
+package de.ids_mannheim.korap.web.service;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.sun.jersey.api.client.ClientResponse;
+
+import de.ids_mannheim.korap.config.KustvaktConfiguration;
+import de.ids_mannheim.korap.config.LiteJerseyTest;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+public class LiteSearchPipeTest extends LiteJerseyTest {
+
+ @Autowired
+ private KustvaktConfiguration config;
+
+ private void setTestPipes () throws IOException {
+ String filename = "test-pipes";
+ File f = new File(filename);
+ if (f.exists()) {
+ f.delete();
+ }
+ f.createNewFile();
+ OutputStreamWriter writer =
+ new OutputStreamWriter(new FileOutputStream(f));
+ writer.append("glemm\t");
+ writer.append(resource().getURI().toString());
+ writer.append(API_VERSION);
+ writer.append("/test/glemm");
+ writer.flush();
+ writer.close();
+
+ config.readPipesFile(filename);
+ }
+
+ @Test
+ public void testSearchWithPipes () throws IOException, KustvaktException {
+ setTestPipes();
+ ClientResponse response = resource().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("pipes", "glemm").get(ClientResponse.class);
+
+ String entity = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+ assertEquals(3, node.at("/query/wrap/key").size());
+ node = node.at("/query/wrap/rewrites");
+ assertEquals(2, node.size());
+ assertEquals("Glemm", node.at("/0/src").asText());
+ assertEquals("operation:override", node.at("/0/operation").asText());
+ assertEquals("key", node.at("/0/scope").asText());
+
+ assertEquals("Kustvakt", node.at("/1/src").asText());
+ assertEquals("operation:injection", node.at("/1/operation").asText());
+ assertEquals("foundry", node.at("/1/scope").asText());
+ }
+}