Implemented predefined VC caching and added VC reference tests.
Change-Id: I84ad56b375f8b076ad92e493775993fc0580249f
diff --git a/core/Changes b/core/Changes
index c89564b..80ee5c4 100644
--- a/core/Changes
+++ b/core/Changes
@@ -1,3 +1,7 @@
+version 0.61.0
+02/08/2018
+ - Updated Krill and Koral versions enabling VC caching and referencing (margaretha)
+
version 0.60.5
09/07/2018
- Fixed status codes (margaretha)
diff --git a/core/pom.xml b/core/pom.xml
index 0fa8b97..01dbacd 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-core</artifactId>
- <version>0.60.5</version>
+ <version>0.61.0</version>
<properties>
<java.version>1.8</java.version>
@@ -223,7 +223,7 @@
<dependency>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Koral</artifactId>
- <version>0.29</version>
+ <version>0.30</version>
<exclusions>
<exclusion>
<groupId>org.eclipse.jetty</groupId>
@@ -275,7 +275,7 @@
<dependency>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Krill</artifactId>
- <version>0.57.0</version>
+ <version>0.58.0</version>
<exclusions>
<exclusion>
<groupId>org.xerial</groupId>
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/KustvaktBaseServer.java b/core/src/main/java/de/ids_mannheim/korap/web/KustvaktBaseServer.java
index d0407ee..2b5694b 100644
--- a/core/src/main/java/de/ids_mannheim/korap/web/KustvaktBaseServer.java
+++ b/core/src/main/java/de/ids_mannheim/korap/web/KustvaktBaseServer.java
@@ -54,9 +54,6 @@
System.out.println(b.toString());
System.out.println();
return (KustvaktArgs) null;
- case "--init":
- kargs.init = true;
- break;
}
}
return kargs;
@@ -112,15 +109,11 @@
@Getter
private String springConfig;
private int port;
- private String[] rootPackages;
- private boolean init;
public KustvaktArgs () {
this.port = -1;
this.springConfig = null;
- this.init = false;
}
-
}
}
diff --git a/full/Changes b/full/Changes
index 079302b..081208d 100644
--- a/full/Changes
+++ b/full/Changes
@@ -1,3 +1,8 @@
+version 0.61.0
+02/08/2018
+ - Added VC referencing tests (margaretha)
+ - Implemented loading and caching named VCs (margaretha)
+
version 0.60.5
02/08/2018
- Added service layer to the search controller (margaretha)
diff --git a/full/pom.xml b/full/pom.xml
index 7ea7aa8..4dac5d4 100644
--- a/full/pom.xml
+++ b/full/pom.xml
@@ -3,7 +3,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-full</artifactId>
- <version>0.60.5</version>
+ <version>0.61.0</version>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
@@ -88,6 +88,7 @@
<excludes>
<exclude>de/ids_mannheim/korap/authentication/*.java</exclude>
<exclude>de/ids_mannheim/korap/web/controller/TokenExpiryTest.java</exclude>
+ <exclude>**/VCReferenceTest.java</exclude>
</excludes>
<includes>
<include>de/ids_mannheim/korap/**/*.java</include>
@@ -150,7 +151,7 @@
<dependency>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-core</artifactId>
- <version>0.60.5</version>
+ <version>0.61.0</version>
</dependency>
<!-- LDAP -->
<dependency>
@@ -214,6 +215,25 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-embedded-glassfish</artifactId>
+ <version>${jersey.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-external</artifactId>
+ <version>${jersey.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>com.sun.jersey.jersey-test-framework</groupId>
+ <artifactId>jersey-test-framework-grizzly2</artifactId>
+ <version>1.19.4</version>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-json</artifactId>
<version>${jersey.version}</version>
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/FullConfiguration.java b/full/src/main/java/de/ids_mannheim/korap/config/FullConfiguration.java
index fc86217..296be80 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/FullConfiguration.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/FullConfiguration.java
@@ -87,6 +87,8 @@
private RSAPrivateKey rsaPrivateKey;
private JWKSet publicKeySet;
private String rsaKeyId;
+
+ private String namedVCPath;
public FullConfiguration (Properties properties) throws Exception {
super(properties);
@@ -109,6 +111,9 @@
setOAuth2Configuration(properties);
setOpenIdConfiguration(properties);
setRSAKeys(properties);
+
+ setNamedVCPath(properties
+ .getProperty("krill.namedVC", "vc"));
}
private void setSecurityConfiguration (Properties properties) {
@@ -648,4 +653,12 @@
public void setMessageDigestAlgorithm (String messageDigestAlgorithm) {
this.messageDigestAlgorithm = messageDigestAlgorithm;
}
+
+ public String getNamedVCPath () {
+ return namedVCPath;
+ }
+
+ public void setNamedVCPath (String namedVCPath) {
+ this.namedVCPath = namedVCPath;
+ }
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java b/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
index e730a0b..c204558 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
@@ -1,31 +1,42 @@
package de.ids_mannheim.korap.config;
-import java.util.Arrays;
+import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
+import javax.annotation.PostConstruct;
+
+import org.springframework.beans.factory.annotation.Autowired;
+
import de.ids_mannheim.korap.oauth2.constant.OAuth2Scope;
import de.ids_mannheim.korap.oauth2.dao.AccessScopeDao;
/**
- * Initializes values in the database from kustvakt configuration.
+ * Initializes values in the database from kustvakt configuration and
+ * performs named VC caching.
*
* @author margaretha
*
*/
public class Initializator {
+ @Autowired
private AccessScopeDao accessScopeDao;
+ @Autowired
+ private NamedVCLoader loader;
- public Initializator (AccessScopeDao accessScopeDao) {
- this.accessScopeDao = accessScopeDao;
+ public Initializator () {}
+
+ public void init () throws IOException {
+ setInitialAccessScope();
+ loader.loadVCToCache();
}
- public void init () {
- setAccessScope();
+ public void initTest () {
+ setInitialAccessScope();
}
-
- private void setAccessScope () {
+
+ private void setInitialAccessScope () {
OAuth2Scope[] enums = OAuth2Scope.values();
Set<String> scopes = new HashSet<>(enums.length);
for (OAuth2Scope s : enums) {
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java b/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
new file mode 100644
index 0000000..043f77e
--- /dev/null
+++ b/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
@@ -0,0 +1,58 @@
+package de.ids_mannheim.korap.config;
+
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import de.ids_mannheim.korap.KrillCollection;
+import de.ids_mannheim.korap.web.SearchKrill;
+
+@Component
+public class NamedVCLoader {
+ @Autowired
+ private FullConfiguration config;
+ @Autowired
+ private SearchKrill searchKrill;
+
+ private static Logger jlog = LogManager.getLogger(NamedVCLoader.class);
+
+ public void loadVCToCache ()
+ throws IOException {
+
+ String dir = config.getNamedVCPath();
+ File d = new File(dir);
+ if (!d.isDirectory()) {
+ throw new IOException("Directory " + dir + " is not valid");
+ }
+
+ for (File file : d.listFiles()) {
+ if (!file.exists()) {
+ throw new IOException("File " + file + " is not found.");
+ }
+ else if (!file.getName().endsWith(".jsonld")) {
+ throw new IOException("File " + file
+ + " is not allowed. Filename must ends with .jsonld");
+ }
+
+ long start = System.currentTimeMillis();
+ String json = FileUtils.readFileToString(file, "utf-8");
+ KrillCollection collection = new KrillCollection(json);
+ collection.setIndex(searchKrill.getIndex());
+
+ String filename = file.getName();
+ filename = filename.substring(0,filename.length()-7);
+ if (collection != null) {
+ collection.storeInCache(filename);
+ }
+ long end = System.currentTimeMillis();
+ jlog.debug(filename + " duration: " + (end - start));
+ jlog.debug("memory cache: "
+ + KrillCollection.cache.calculateInMemorySize());
+ }
+ }
+}
diff --git a/full/src/main/java/de/ids_mannheim/korap/server/KustvaktServer.java b/full/src/main/java/de/ids_mannheim/korap/server/KustvaktServer.java
index 35879c8..87f1e0a 100644
--- a/full/src/main/java/de/ids_mannheim/korap/server/KustvaktServer.java
+++ b/full/src/main/java/de/ids_mannheim/korap/server/KustvaktServer.java
@@ -3,7 +3,6 @@
import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
-import java.net.URL;
import java.util.Properties;
import de.ids_mannheim.korap.config.FullConfiguration;
@@ -17,8 +16,6 @@
*/
public class KustvaktServer extends KustvaktBaseServer {
- private static FullConfiguration fullConfig;
-
public static final String API_VERSION = "v0.1";
public static void main (String[] args) throws Exception {
@@ -38,8 +35,8 @@
properties.load(in);
in.close();
- fullConfig = new FullConfiguration(properties);
- config = fullConfig;
+
+ config = new FullConfiguration(properties);
if (kargs == null)
System.exit(0);
@@ -47,8 +44,6 @@
if (kargs.getSpringConfig() == null){
kargs.setSpringConfig("default-config.xml");
}
- kargs.setRootPackages(new String[] { "de.ids_mannheim.korap.web.utils",
- "de.ids_mannheim.korap.web.service.full" });
rootPackages = "de.ids_mannheim.korap.web.utils;"
+ "de.ids_mannheim.korap.web.service.full";
server.start();
diff --git a/full/src/main/resources/default-config.xml b/full/src/main/resources/default-config.xml
index 90ea568..32ebb35 100644
--- a/full/src/main/resources/default-config.xml
+++ b/full/src/main/resources/default-config.xml
@@ -168,7 +168,6 @@
<bean id="initializator" class="de.ids_mannheim.korap.config.Initializator"
init-method="init">
- <constructor-arg name="accessScopeDao" ref="accessScopeDao" />
</bean>
<!-- Krill -->
diff --git a/full/src/main/resources/ehcache.xml b/full/src/main/resources/ehcache.xml
index d2be647..8cddd0e 100644
--- a/full/src/main/resources/ehcache.xml
+++ b/full/src/main/resources/ehcache.xml
@@ -1,5 +1,6 @@
-<ehcache xmlns:xsi='http://www.w3.org/2001/XMLSchema-instance'
- xsi:schemaLocation='http://ehcache.org/ehcache.xsd'>
+<ehcache xsi:noNamespaceSchemaLocation="http://www.ehcache.org/ehcache.xsd"
+ updateCheck="true" monitoring="autodetect" dynamicConfig="true">
+
<defaultCache eternal='true' overflowToDisk='false'/>
<!--maxBytesLocalHeap="200M"-->
<diskStore path="./cache_store"/>
@@ -50,5 +51,14 @@
eternal='false'
memoryStoreEvictionPolicy="LRU"
maxEntriesLocalHeap="500"
- overflowToDisk='false'/>
+ overflowToDisk='false'/>
+
+ <cache name="named_vc"
+ eternal="true"
+ memoryStoreEvictionPolicy="LRU"
+ maxBytesLocalHeap="256M"
+ maxBytesLocalDisk="1G"
+ diskExpiryThreadIntervalSeconds = "120" >
+ <persistence strategy="localTempSwap"/>
+ </cache>
</ehcache>
diff --git a/full/src/main/resources/kustvakt.conf b/full/src/main/resources/kustvakt.conf
index 22e7e14..d7fbf7d 100644
--- a/full/src/main/resources/kustvakt.conf
+++ b/full/src/main/resources/kustvakt.conf
@@ -5,6 +5,8 @@
krill.index.commit.log = log/krill.commit.log
krill.index.commit.auto = 500
krill.index.relations.max = 100
+## Directory path of virtual corpora to cache
+krill.namedVC = vc
## LDAP
ldap.config = file-path-to-ldap-config
diff --git a/full/src/test/java/de/ids_mannheim/korap/web/controller/OAuth2ControllerTest.java b/full/src/test/java/de/ids_mannheim/korap/web/controller/OAuth2ControllerTest.java
index 7fb5d4c..a5ba179 100644
--- a/full/src/test/java/de/ids_mannheim/korap/web/controller/OAuth2ControllerTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/web/controller/OAuth2ControllerTest.java
@@ -8,7 +8,6 @@
import javax.ws.rs.core.MultivaluedMap;
import javax.ws.rs.core.Response.Status;
-import org.apache.commons.codec.binary.Base64;
import org.apache.http.entity.ContentType;
import org.apache.oltu.oauth2.common.error.OAuthError;
import org.apache.oltu.oauth2.common.message.types.GrantType;
@@ -187,6 +186,7 @@
tokenForm.add("client_id", "fCBbQkAyYzI4NzUxMg");
tokenForm.add("client_secret", "secret");
tokenForm.add("code", code);
+ System.out.println(code);
response = requestToken(tokenForm);
String entity = response.getEntity(String.class);
diff --git a/full/src/test/java/de/ids_mannheim/korap/web/controller/VCReferenceTest.java b/full/src/test/java/de/ids_mannheim/korap/web/controller/VCReferenceTest.java
new file mode 100644
index 0000000..ae3d38e
--- /dev/null
+++ b/full/src/test/java/de/ids_mannheim/korap/web/controller/VCReferenceTest.java
@@ -0,0 +1,75 @@
+package de.ids_mannheim.korap.web.controller;
+
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.sun.jersey.api.client.ClientResponse;
+
+import de.ids_mannheim.korap.config.SpringJerseyTest;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+public class VCReferenceTest extends SpringJerseyTest {
+
+ @Test
+ public void testVCRef () throws KustvaktException {
+ testSearchWithoutVCRefOr();
+ testSearchWithoutVCRefAnd();
+ // auto caching
+ testSearchWithVCRefEqual();
+ testSearchWithVCRefNotEqual();
+ // retrieve from cache
+ testSearchWithVCRefEqual();
+ testSearchWithVCRefNotEqual();
+ }
+
+ private void testSearchWithoutVCRefOr () throws KustvaktException {
+ ClientResponse response = resource().path("search")
+ .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("cq",
+ "textSigle=\"GOE/AGF/00000\" | textSigle=\"GOE/AGA/01784\"")
+ .get(ClientResponse.class);
+
+ String ent = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(ent);
+ assertTrue(node.at("/matches").size() > 0);
+ }
+
+ private void testSearchWithoutVCRefAnd () throws KustvaktException {
+ ClientResponse response = resource().path("search")
+ .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("cq",
+ "textSigle!=\"GOE/AGI/04846\" & textSigle!=\"GOE/AGA/01784\"")
+ .get(ClientResponse.class);
+
+ String ent = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(ent);
+ assertTrue(node.at("/matches").size() > 0);
+ }
+
+ public void testSearchWithVCRefEqual () throws KustvaktException {
+ ClientResponse response = resource().path("search")
+ .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("cq", "referTo named-vc1")
+ .get(ClientResponse.class);
+
+ String ent = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(ent);
+ assertTrue(node.at("/matches").size() > 0);
+
+ }
+
+ public void testSearchWithVCRefNotEqual () throws KustvaktException {
+ ClientResponse response = resource().path("search")
+ .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("cq", "referTo named-vc2")
+ .get(ClientResponse.class);
+
+ String ent = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(ent);
+ assertTrue(node.at("/matches").size() > 0);
+ }
+
+}
diff --git a/full/src/test/resources/kustvakt-test.conf b/full/src/test/resources/kustvakt-test.conf
index 694abce..8690e4f 100644
--- a/full/src/test/resources/kustvakt-test.conf
+++ b/full/src/test/resources/kustvakt-test.conf
@@ -5,6 +5,8 @@
krill.index.commit.log = log/krill.commit.log
krill.index.commit.auto = 500
krill.index.relations.max = 100
+## Directory path of virtual corpora to cache
+krill.namedVC = vc
#LDAP
ldap.config = file-path-to-ldap-config
diff --git a/full/src/test/resources/test-config.xml b/full/src/test/resources/test-config.xml
index ad3d702..f438fd2 100644
--- a/full/src/test/resources/test-config.xml
+++ b/full/src/test/resources/test-config.xml
@@ -167,8 +167,7 @@
</bean>
<bean id="initializator" class="de.ids_mannheim.korap.config.Initializator"
- init-method="init">
- <constructor-arg name="accessScopeDao" ref="accessScopeDao" />
+ init-method="initTest">
</bean>
diff --git a/lite/pom.xml b/lite/pom.xml
index 9b3e8dd..103a225 100644
--- a/lite/pom.xml
+++ b/lite/pom.xml
@@ -147,7 +147,7 @@
<dependency>
<groupId>de.ids_mannheim.korap</groupId>
<artifactId>Kustvakt-core</artifactId>
- <version>0.60.5</version>
+ <version>0.61.0</version>
</dependency>
<!-- Spring -->
diff --git a/lite/src/main/java/de/ids_mannheim/korap/server/KustvaktLiteServer.java b/lite/src/main/java/de/ids_mannheim/korap/server/KustvaktLiteServer.java
index 4be47cd..66329ef 100644
--- a/lite/src/main/java/de/ids_mannheim/korap/server/KustvaktLiteServer.java
+++ b/lite/src/main/java/de/ids_mannheim/korap/server/KustvaktLiteServer.java
@@ -33,8 +33,6 @@
config = new KustvaktConfiguration(properties);
kargs.setSpringConfig("lite-config.xml");
- kargs.setRootPackages(
- new String[] { "de.ids_mannheim.korap.web.service.lite" });
rootPackages = "de.ids_mannheim.korap.web.service.lite";
server.start();