Implemented pipe extension in the search API.
Change-Id: If2a486185a7d16a27b7b46d35d85b7d5f27b66cd
diff --git a/core/Changes b/core/Changes
index 272c8d1..64c0651 100644
--- a/core/Changes
+++ b/core/Changes
@@ -1,3 +1,7 @@
+# version 0.62.3
+03/12/2019
+ - Implemented pipe extension in the search API (margaretha)
+
# version 0.62.2
13/11/2019
- Added warnings when requesting non-public fields via the search API with
diff --git a/core/pom.xml b/core/pom.xml
index 7bef3cd..be3b949 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-core</artifactId>
- <version>0.62.2</version>
+ <version>0.62.3</version>
<properties>
<java.version>1.8</java.version>
@@ -210,7 +210,7 @@
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
- <version>1.7.25</version>
+ <version>1.7.29</version>
</dependency>
<!-- Java Assist -->
diff --git a/core/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java b/core/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
index 8144124..dcbc3cb 100644
--- a/core/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
+++ b/core/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
@@ -1,7 +1,11 @@
package de.ids_mannheim.korap.config;
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
+import java.io.InputStreamReader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
@@ -12,6 +16,8 @@
import java.util.Set;
import java.util.regex.Pattern;
+import org.apache.commons.io.FileUtils;
+
import de.ids_mannheim.korap.util.KrillProperties;
import de.ids_mannheim.korap.utils.TimeUtils;
import lombok.Getter;
@@ -100,8 +106,11 @@
// another variable might be needed to define which metadata fields are restricted
private boolean isMetadataRestricted = false;
+ public static Map<String, String> pipes = new HashMap<>();
+
public KustvaktConfiguration (Properties properties) throws Exception {
load(properties);
+ readPipesFile("pipes");
KrillProperties.setProp(properties);
}
@@ -191,6 +200,27 @@
// properties.getProperty("security.passcode.salt",
// "accountCreation");
}
+
+ public void readPipesFile (String filename) throws IOException {
+ File file = new File(filename);
+ if (file.exists()) {
+ BufferedReader br = new BufferedReader(
+ new InputStreamReader(new FileInputStream(file)));
+
+ String line = null;
+ while( (line=br.readLine())!=null ){
+ String[] parts = line.split("\t");
+ if (parts.length !=2){
+ continue;
+ }
+ else{
+ pipes.put(parts[0], parts[1]);
+ }
+ }
+ br.close();
+ }
+ }
+
/**
* set properties
diff --git a/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java b/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
index 3a04452..46452ca 100644
--- a/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
+++ b/core/src/main/java/de/ids_mannheim/korap/service/SearchService.java
@@ -8,6 +8,7 @@
import javax.annotation.PostConstruct;
import javax.servlet.ServletContext;
import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.UriBuilder;
@@ -17,6 +18,9 @@
import org.springframework.stereotype.Service;
import com.fasterxml.jackson.databind.JsonNode;
+import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.WebResource;
import com.sun.jersey.core.util.MultivaluedMapImpl;
import de.ids_mannheim.de.init.VCLoader;
@@ -120,7 +124,7 @@
@SuppressWarnings("unchecked")
public String search (String engine, String username, HttpHeaders headers,
String q, String ql, String v, String cq, String fields,
- Integer pageIndex, Integer pageInteger, String ctx,
+ String pipes, Integer pageIndex, Integer pageInteger, String ctx,
Integer pageLength, Boolean cutoff, boolean accessRewriteDisabled)
throws KustvaktException {
@@ -129,6 +133,11 @@
"page must start from 1", "page");
}
+ String[] pipeArray = null;
+ if (pipes!=null && !pipes.isEmpty()){
+ pipeArray = pipes.split(",");
+ }
+
KustvaktConfiguration.BACKENDS eng = this.config.chooseBackend(engine);
User user = createUser(username, headers);
CorpusAccess corpusAccess = user.getCorpusAccess();
@@ -157,8 +166,10 @@
throw new KustvaktException(serializer.toJSON());
}
- String query =
- this.rewriteHandler.processQuery(serializer.toJSON(), user);
+ String query = serializer.toJSON();
+ query = runPipes(query,pipeArray);
+
+ query = this.rewriteHandler.processQuery(query, user);
if (DEBUG){
jlog.debug("the serialized query " + query);
}
@@ -175,6 +186,23 @@
}
+ private String runPipes (String query, String[] pipeArray) {
+ if (pipeArray !=null){
+ for (int i=0; i<pipeArray.length; i++){
+ String url = KustvaktConfiguration.pipes.get(pipeArray[i]);
+ // update query by sending it to a pipe URL
+ // NOTE: request formulation may vary depending on the service
+ Client client = Client.create();
+ WebResource resource = client.resource(url);
+ ClientResponse response =
+ resource.type(MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class, query);
+ query = response.getEntity(String.class);
+ }
+ }
+ return query;
+ }
+
private void handleNonPublicFields (List<String> fieldList,
boolean accessRewriteDisabled, QuerySerializer serializer) {
List<String> nonPublicFields = new ArrayList<>();
diff --git a/core/src/main/java/de/ids_mannheim/korap/test/TestController.java b/core/src/main/java/de/ids_mannheim/korap/test/TestController.java
new file mode 100644
index 0000000..43144bb
--- /dev/null
+++ b/core/src/main/java/de/ids_mannheim/korap/test/TestController.java
@@ -0,0 +1,33 @@
+package de.ids_mannheim.korap.test;
+
+import java.io.InputStream;
+
+import javax.ws.rs.Produces;
+import javax.ws.rs.Consumes;
+import javax.ws.rs.POST;
+import javax.ws.rs.Path;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.Response;
+
+import org.springframework.stereotype.Controller;
+
+/**
+ * Controllers used only for testing
+ *
+ * @author margaretha
+ *
+ */
+@Controller
+@Path("/{version}/test")
+@Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
+public class TestController {
+
+ @POST
+ @Path("glemm")
+ @Consumes(MediaType.APPLICATION_JSON)
+ public Response dummyGlemm (String jsonld) {
+ InputStream is = getClass().getClassLoader()
+ .getResourceAsStream("test-pipes.jsonld");
+ return Response.ok(is).build();
+ }
+}
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java b/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
index 8ccb8f1..76bf89d 100644
--- a/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
+++ b/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
@@ -136,6 +136,40 @@
}
}
+ /** Performs for the given query
+ *
+ * @param securityContext
+ * @param request
+ * @param headers
+ * @param locale
+ * @param q
+ * query
+ * @param ql
+ * query language
+ * @param v
+ * query language version
+ * @param ctx
+ * result context
+ * @param cutoff
+ * determines to limit search results to one page only
+ * or not (default false)
+ * @param pageLength
+ * the number of results should be included in a page
+ * @param pageIndex
+ * @param pageInteger page number
+ * @param fields
+ * metadata fields to be included, separated by comma
+ * @param pipes
+ * external plugins for additional processing,
+ * separated by comma
+ * @param accessRewriteDisabled
+ * determine if access rewrite should be disabled
+ * (default false)
+ * @param cq
+ * corpus query defining a virtual corpus
+ * @param engine
+ * @return search results in JSON
+ */
@GET
@Path("{version}/search")
public Response searchGet (@Context SecurityContext securityContext,
@@ -148,6 +182,7 @@
@QueryParam("offset") Integer pageIndex,
@QueryParam("page") Integer pageInteger,
@QueryParam("fields") String fields,
+ @QueryParam("pipes") String pipes,
@QueryParam("access-rewrite-disabled") boolean accessRewriteDisabled,
@QueryParam("cq") String cq,
@QueryParam("engine") String engine) {
@@ -159,8 +194,9 @@
try {
scopeService.verifyScope(context, OAuth2Scope.SEARCH);
result = searchService.search(engine, context.getUsername(),
- headers, q, ql, v, cq, fields, pageIndex, pageInteger, ctx,
- pageLength, cutoff, accessRewriteDisabled);
+ headers, q, ql, v, cq, fields, pipes, pageIndex,
+ pageInteger, ctx, pageLength, cutoff,
+ accessRewriteDisabled);
}
catch (KustvaktException e) {
throw kustvaktResponseHandler.throwit(e);
diff --git a/core/src/main/resources/test-pipes.jsonld b/core/src/main/resources/test-pipes.jsonld
new file mode 100644
index 0000000..eca8cec
--- /dev/null
+++ b/core/src/main/resources/test-pipes.jsonld
@@ -0,0 +1,26 @@
+{
+ "meta": {
+ "snippets": true,
+ "timeout": 10000
+ },
+ "query": {
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "match": "match:eq",
+ "key": [
+ "der",
+ "die",
+ "das"
+ ],
+ "layer": "orth",
+ "rewrites": [{
+ "@type": "koral:rewrite",
+ "src": "Glemm",
+ "operation": "operation:override",
+ "scope": "key"
+ }]
+ }
+ },
+ "@context": "http://korap.ids-mannheim.de/ns/koral/0.3/context.jsonld"
+}