Add pipe response rewriting (#794)
limited to korap host by default. Configurable via kustvakt.conf.
Change-Id: Ifb0b6470a4b1fc6db79b3bb05443f6482f4b7ea8
diff --git a/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java b/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
index 658208e..d442741 100644
--- a/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
+++ b/src/main/java/de/ids_mannheim/korap/config/KustvaktConfiguration.java
@@ -125,6 +125,10 @@
// EM: Maybe needed when we support pipe registration
@Deprecated
public static Map<String, String> pipes = new HashMap<>();
+
+ // EM: default host allowed for piping query and response
+ // (temporary solution without pipe registration)
+ private String pipeHost;
public KustvaktConfiguration (Properties properties) throws Exception {
load(properties);
@@ -240,6 +244,8 @@
loginTimeout = Integer.parseInt(properties.getProperty(
"timeout.login", "90000"));
+ pipeHost = properties.getProperty("pipe.host",
+ "https://korap.ids-mannheim.de");
}
@Deprecated
diff --git a/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java b/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
index eeddaba..f5bce26 100644
--- a/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
+++ b/src/main/java/de/ids_mannheim/korap/core/service/SearchService.java
@@ -61,6 +61,9 @@
@Autowired
protected RewriteHandler rewriteHandler;
+ @Autowired
+ protected KustvaktConfiguration config;
+
@PostConstruct
private void doPostConstruct () {
UriBuilder builder = UriBuilder.fromUri("http://10.0.10.13").port(9997);
@@ -122,22 +125,18 @@
}
@SuppressWarnings("unchecked")
- public String search (String engine, String username, HttpHeaders headers,
- String q, String ql, String v, List<String> cqList, String fields,
- String pipes, Integer pageIndex, Integer pageInteger, String ctx,
- Integer pageLength, Boolean cutoff, boolean accessRewriteDisabled,
- boolean showTokens, boolean showSnippet) throws KustvaktException {
+ public String search (String engine, String username, HttpHeaders headers,
+ String q, String ql, String v, List<String> cqList, String fields,
+ String pipes, String responsePipes, Integer pageIndex,
+ Integer pageInteger, String ctx, Integer pageLength, Boolean cutoff,
+ boolean accessRewriteDisabled, boolean showTokens,
+ boolean showSnippet) throws KustvaktException {
if (pageInteger != null && pageInteger < 1) {
throw new KustvaktException(StatusCodes.INVALID_ARGUMENT,
"page must start from 1", "page");
}
- String[] pipeArray = null;
- if (pipes != null && !pipes.isEmpty()) {
- pipeArray = pipes.split(",");
- }
-
User user = createUser(username, headers);
CorpusAccess corpusAccess = user.getCorpusAccess();
@@ -178,7 +177,8 @@
query = addWarning(query, warning);
}
- query = runPipes(query, pipeArray);
+ // Query pipe rewrite
+ query = runPipes(query, pipes);
query = this.rewriteHandler.processQuery(query, user);
if (DEBUG) {
@@ -204,7 +204,7 @@
result = searchKrill.search(query);
}
// jlog.debug("Query result: " + result);
-
+
if (config.isTotalResultCacheEnabled()) {
result = afterCheckTotalResultCache(hashedKoralQuery, result);
}
@@ -212,10 +212,13 @@
if (!hasCutOff) {
result = removeCutOff(result);
}
+
+ // Response pipe rewrite
+ result = runPipes(result, responsePipes);
return result;
}
-
+
private String removeCutOff (String result) throws KustvaktException {
ObjectNode resultNode = (ObjectNode) JsonUtils.readTree(result);
ObjectNode meta = (ObjectNode) resultNode.at("/meta");
@@ -291,58 +294,66 @@
*
* @param query
* the original koral query
- * @param pipeArray
+ * @param pipes
* the pipe service URLs
* @param serializer
* the query serializer
* @return a modified koral query
* @throws KustvaktException
*/
- private String runPipes (String query, String[] pipeArray)
+ private String runPipes (String query, String pipes)
throws KustvaktException {
- if (pipeArray != null) {
+ if (pipes != null && !pipes.isEmpty()) {
+ String[] pipeArray = pipes.split(",");
+
for (int i = 0; i < pipeArray.length; i++) {
String pipeURL = pipeArray[i];
- try {
- URL url = new URL(pipeURL);
- HttpURLConnection connection = (HttpURLConnection) url
- .openConnection();
- connection.setRequestMethod("POST");
- connection.setRequestProperty("Content-Type",
- "application/json; charset=UTF-8");
- connection.setRequestProperty("Accept", "application/json");
- connection.setDoOutput(true);
- OutputStream os = connection.getOutputStream();
- byte[] input = query.getBytes("utf-8");
- os.write(input, 0, input.length);
-
- String entity = null;
- if (connection.getResponseCode() == HttpStatus.SC_OK) {
- BufferedReader br = new BufferedReader(
- new InputStreamReader(
- connection.getInputStream(), "utf-8"));
- StringBuilder response = new StringBuilder();
- String responseLine = null;
- while ((responseLine = br.readLine()) != null) {
- response.append(responseLine.trim());
+ if (pipeURL.startsWith(config.getPipeHost())) {
+ try {
+ URL url = new URL(pipeURL);
+ HttpURLConnection connection = (HttpURLConnection) url
+ .openConnection();
+ connection.setRequestMethod("POST");
+ connection.setRequestProperty("Content-Type",
+ "application/json; charset=UTF-8");
+ connection.setRequestProperty("Accept", "application/json");
+ connection.setDoOutput(true);
+ OutputStream os = connection.getOutputStream();
+ byte[] input = query.getBytes("utf-8");
+ os.write(input, 0, input.length);
+
+ String entity = null;
+ if (connection.getResponseCode() == HttpStatus.SC_OK) {
+ BufferedReader br = new BufferedReader(
+ new InputStreamReader(
+ connection.getInputStream(), "utf-8"));
+ StringBuilder response = new StringBuilder();
+ String responseLine = null;
+ while ((responseLine = br.readLine()) != null) {
+ response.append(responseLine.trim());
+ }
+ entity = response.toString();
}
- entity = response.toString();
+
+ if (entity != null && !entity.isEmpty()) {
+ query = entity;
+ }
+ else {
+ query = handlePipeError(query, pipeURL,
+ connection.getResponseCode() + " "
+ + connection.getResponseMessage());
+ }
}
-
- if (entity != null && !entity.isEmpty()) {
- query = entity;
- }
- else {
- query = handlePipeError(query, pipeURL,
- connection.getResponseCode() + " "
- + connection.getResponseMessage());
+ catch (Exception e) {
+ query = handlePipeError(query, pipeURL, e.getMessage());
}
}
- catch (Exception e) {
- query = handlePipeError(query, pipeURL, e.getMessage());
+ else {
+ query = handlePipeError(query, pipeURL,
+ "Unrecognized pipe URL");
}
}
- }
+ }
return query;
}
@@ -361,8 +372,14 @@
private String addWarning (String query, JsonNode warning)
throws KustvaktException {
-
- ObjectNode node = (ObjectNode) JsonUtils.readTree(query);
+ ObjectNode node = null;
+ try {
+ node = (ObjectNode) JsonUtils.readTree(query);
+ }
+ catch (Exception e) {
+ throw new KustvaktException(StatusCodes.DESERIALIZATION_FAILED,
+ "Invalid JSON format");
+ }
if (node.has("warnings")) {
warning = warning.at("/warnings/0");
ArrayNode arrayNode = (ArrayNode) node.get("warnings");
diff --git a/src/main/java/de/ids_mannheim/korap/core/web/controller/SearchController.java b/src/main/java/de/ids_mannheim/korap/core/web/controller/SearchController.java
index 0838cf0..19aea8e 100644
--- a/src/main/java/de/ids_mannheim/korap/core/web/controller/SearchController.java
+++ b/src/main/java/de/ids_mannheim/korap/core/web/controller/SearchController.java
@@ -226,6 +226,7 @@
@QueryParam("page") Integer pageInteger,
@QueryParam("fields") String fields,
@QueryParam("pipes") String pipes,
+ @QueryParam("response-pipes") String responsePipes,
@QueryParam("access-rewrite-disabled") boolean accessRewriteDisabled,
@QueryParam("show-tokens") boolean showTokens,
@DefaultValue("true") @QueryParam("show-snippet") boolean showSnippet,
@@ -239,9 +240,9 @@
try {
scopeService.verifyScope(context, OAuth2Scope.SEARCH);
result = searchService.search(engine, context.getUsername(),
- headers, q, ql, v, cq, fields, pipes, pageIndex,
- pageInteger, ctx, pageLength, cutoff, accessRewriteDisabled,
- showTokens, showSnippet);
+ headers, q, ql, v, cq, fields, pipes, responsePipes,
+ pageIndex, pageInteger, ctx, pageLength, cutoff,
+ accessRewriteDisabled, showTokens, showSnippet);
}
catch (KustvaktException e) {
throw kustvaktResponseHandler.throwit(e);
diff --git a/src/main/resources/kustvakt-lite.conf b/src/main/resources/kustvakt-lite.conf
index 81c9fe6..eaa8cd5 100644
--- a/src/main/resources/kustvakt-lite.conf
+++ b/src/main/resources/kustvakt-lite.conf
@@ -21,6 +21,9 @@
# default
kustvakt.base.url=/api/*
+# Pipe (Please change or commented for production!)
+pipe.host = http://localhost
+
# default foundries for layers
default.foundry.partOfSpeech = tt
default.foundry.lemma = tt
diff --git a/src/test/java/de/ids_mannheim/korap/config/LiteJerseyTest.java b/src/test/java/de/ids_mannheim/korap/config/LiteJerseyTest.java
index 80b00a5..f48b8b3 100644
--- a/src/test/java/de/ids_mannheim/korap/config/LiteJerseyTest.java
+++ b/src/test/java/de/ids_mannheim/korap/config/LiteJerseyTest.java
@@ -22,6 +22,9 @@
public static final String API_VERSION = "v1.0";
+ protected final static String freeCorpusAccess = "Free corpus access policy "
+ + "has been added.";
+
@Autowired
protected GenericApplicationContext applicationContext;
diff --git a/src/test/java/de/ids_mannheim/korap/web/controller/SearchPipeTest.java b/src/test/java/de/ids_mannheim/korap/web/controller/SearchPipeTest.java
index 8f58c16..6b3dc7f 100644
--- a/src/test/java/de/ids_mannheim/korap/web/controller/SearchPipeTest.java
+++ b/src/test/java/de/ids_mannheim/korap/web/controller/SearchPipeTest.java
@@ -124,14 +124,14 @@
"operation:injection");
assertEquals(freeCorpusAccess,
node.at("/collection/rewrites/0/_comment").asText());
- node = node.at("/query/wrap/rewrites");
- assertEquals(2, node.size());
- assertEquals(node.at("/0/src").asText(), "Glemm");
- assertEquals(node.at("/0/operation").asText(), "operation:override");
- assertEquals(node.at("/0/scope").asText(), "key");
- assertEquals(node.at("/1/src").asText(), "Kustvakt");
- assertEquals(node.at("/1/operation").asText(), "operation:injection");
- assertEquals(node.at("/1/scope").asText(), "foundry");
+ node = node.at("/query/wrap/rewrites");
+ assertEquals(2, node.size());
+ assertEquals("Glemm", node.at("/0/src").asText());
+ assertEquals("operation:override", node.at("/0/operation").asText());
+ assertEquals("key", node.at("/0/scope").asText());
+ assertEquals("Kustvakt", node.at("/1/src").asText());
+ assertEquals("operation:injection", node.at("/1/operation").asText());
+ assertEquals("foundry", node.at("/1/scope").asText());
}
@Test
@@ -213,7 +213,7 @@
String entity = response.readEntity(String.class);
JsonNode node = JsonUtils.readTree(entity);
assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
- assertEquals(node.at("/warnings/0/3").asText(), "404 Not Found");
+ assertEquals("404 Not Found", node.at("/warnings/0/3").asText());
}
@Test
@@ -225,7 +225,7 @@
String entity = response.readEntity(String.class);
JsonNode node = JsonUtils.readTree(entity);
assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
- assertEquals(node.at("/warnings/0/3").asText(), "glemm");
+ assertEquals("Unrecognized pipe URL", node.at("/warnings/0/3").asText());
}
@Test
@@ -238,11 +238,11 @@
.queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
.queryParam("pipes", pipeUri).request().get();
String entity = response.readEntity(String.class);
- assertEquals(Status.OK.getStatusCode(), response.getStatus());
- JsonNode node = JsonUtils.readTree(entity);
- assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
- assertEquals(node.at("/warnings/0/3").asText(),
- "415 Unsupported Media Type");
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ JsonNode node = JsonUtils.readTree(entity);
+ assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
+ assertEquals("415 Unsupported Media Type",
+ node.at("/warnings/0/3").asText() );
}
@Test
@@ -259,10 +259,11 @@
assertEquals(2, node.at("/warnings").size());
assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
assertEquals(url, node.at("/warnings/0/2").asText());
- assertEquals(node.at("/warnings/0/3").asText(), "404 Not Found");
- assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/1/0").asInt());
- assertEquals(node.at("/warnings/1/2").asText(), "http://glemm");
- assertEquals(node.at("/warnings/1/3").asText(), "glemm");
+ assertEquals("404 Not Found", node.at("/warnings/0/3").asText());
+ assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/1/0").asInt());
+ assertEquals("http://glemm", node.at("/warnings/1/2").asText());
+ assertEquals("Unrecognized pipe URL",
+ node.at("/warnings/1/3").asText());
}
@Test
diff --git a/src/test/java/de/ids_mannheim/korap/web/lite/LiteSearchPipeTest.java b/src/test/java/de/ids_mannheim/korap/web/lite/LiteSearchPipeTest.java
index 704ae5f..da7b7e9 100644
--- a/src/test/java/de/ids_mannheim/korap/web/lite/LiteSearchPipeTest.java
+++ b/src/test/java/de/ids_mannheim/korap/web/lite/LiteSearchPipeTest.java
@@ -115,14 +115,14 @@
String entity = response.readEntity(String.class);
JsonNode node = JsonUtils.readTree(entity);
assertEquals(2, node.at("/query/wrap/key").size());
- node = node.at("/query/wrap/rewrites");
- assertEquals(2, node.size());
- assertEquals(node.at("/0/src").asText(), "Glemm");
- assertEquals(node.at("/0/operation").asText(), "operation:override");
- assertEquals(node.at("/0/scope").asText(), "key");
- assertEquals(node.at("/1/src").asText(), "Kustvakt");
- assertEquals(node.at("/1/operation").asText(), "operation:injection");
- assertEquals(node.at("/1/scope").asText(), "foundry");
+ node = node.at("/query/wrap/rewrites");
+ assertEquals(2, node.size());
+ assertEquals("Glemm", node.at("/0/src").asText());
+ assertEquals("operation:override", node.at("/0/operation").asText());
+ assertEquals("key", node.at("/0/scope").asText());
+ assertEquals("Kustvakt", node.at("/1/src").asText());
+ assertEquals("operation:injection", node.at("/1/operation").asText());
+ assertEquals("foundry", node.at("/1/scope").asText());
}
@Test
@@ -182,7 +182,7 @@
String entity = response.readEntity(String.class);
JsonNode node = JsonUtils.readTree(entity);
assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
- assertEquals(node.at("/warnings/0/3").asText(), "404 Not Found");
+ assertEquals("404 Not Found", node.at("/warnings/0/3").asText());
}
@Test
@@ -194,7 +194,7 @@
String entity = response.readEntity(String.class);
JsonNode node = JsonUtils.readTree(entity);
assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
- assertEquals(node.at("/warnings/0/3").asText(), "glemm");
+ assertEquals("Unrecognized pipe URL", node.at("/warnings/0/3").asText());
}
@Test
@@ -207,11 +207,11 @@
.queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
.queryParam("pipes", pipeUri).request().get();
String entity = response.readEntity(String.class);
- assertEquals(Status.OK.getStatusCode(), response.getStatus());
- JsonNode node = JsonUtils.readTree(entity);
- assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
- assertEquals(node.at("/warnings/0/3").asText(),
- "415 Unsupported Media Type");
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ JsonNode node = JsonUtils.readTree(entity);
+ assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
+ assertEquals("415 Unsupported Media Type",
+ node.at("/warnings/0/3").asText() );
}
@Test
@@ -228,10 +228,11 @@
assertEquals(2, node.at("/warnings").size());
assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
assertEquals(url, node.at("/warnings/0/2").asText());
- assertEquals(node.at("/warnings/0/3").asText(), "404 Not Found");
- assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/1/0").asInt());
- assertEquals(node.at("/warnings/1/2").asText(), "http://glemm");
- assertEquals(node.at("/warnings/1/3").asText(), "glemm");
+ assertEquals("404 Not Found", node.at("/warnings/0/3").asText());
+ assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/1/0").asInt());
+ assertEquals("http://glemm", node.at("/warnings/1/2").asText());
+ assertEquals("Unrecognized pipe URL",
+ node.at("/warnings/1/3").asText());
}
@Test
@@ -307,4 +308,5 @@
node = JsonUtils.readTree(entity);
assertEquals(StatusCodes.PIPE_FAILED, node.at("/warnings/0/0").asInt());
}
+
}
diff --git a/src/test/resources/kustvakt-lite-test.conf b/src/test/resources/kustvakt-lite-test.conf
new file mode 100644
index 0000000..0143cec
--- /dev/null
+++ b/src/test/resources/kustvakt-lite-test.conf
@@ -0,0 +1,40 @@
+# Krill settings
+
+# index dir
+krill.indexDir= sample-index
+
+krill.index.commit.count = 134217000
+krill.index.commit.log = log/krill.commit.log
+krill.index.commit.auto = 500
+krill.index.relations.max = 100
+
+# krill.namedVC=data/vc
+
+
+# Kustvakt settings
+
+api.welcome.message = Welcome to KorAP API!
+current.api.version = v1.0
+# multiple versions separated by space
+supported.api.version = v1.0
+
+# default
+kustvakt.base.url=/api/*
+
+# server
+server.port=8089
+server.host=localhost
+
+# Pipe (optional)
+# pipe.host=https://korap.ids-mannheim.de (default)
+# LOCALHOST ONLY FOR TESTING
+pipe.host=http://localhost
+
+# default foundries for layers
+default.foundry.partOfSpeech = tt
+default.foundry.lemma = tt
+default.foundry.orthography = opennlp
+default.foundry.dependency = malt
+default.foundry.constituent = corenlp
+default.foundry.morphology = marmot
+default.foundry.surface = base
\ No newline at end of file
diff --git a/src/test/resources/kustvakt-test.conf b/src/test/resources/kustvakt-test.conf
index d40c39f..a048478 100644
--- a/src/test/resources/kustvakt-test.conf
+++ b/src/test/resources/kustvakt-test.conf
@@ -31,6 +31,9 @@
## Cache
cache.total.results.enabled = true
+# Pipe
+pipe.host=http://localhost
+
# Default foundries for specific layers (optional)
#
default.foundry.partOfSpeech = tt