Implemented configurable resource filters for search and match info APIs
Added tests and settings for ICC scenario.
Change-Id: If22b96cd12f2a39c134a45f9e3e4b2da8bcd36dc
diff --git a/.gitignore b/.gitignore
index 1e8daea..1147d0b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,8 @@
/lite/krill_cache/
**/ldap.conf
+icc-index
+
# FB specific Kustvakt.conf
src\main\resources\Kustvakt.conf
/temp/
diff --git a/core/Changes b/core/Changes
index cd92fca..3590c21 100644
--- a/core/Changes
+++ b/core/Changes
@@ -2,6 +2,7 @@
- Support token array in matchinfo (fixes #570; diewald)
- Added user info web-service (solved #566)
+- Implemented configurable resource filters for search and match info APIs
# version 0.69.3
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java b/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
index 3cc768b..7c180e8 100644
--- a/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
+++ b/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
@@ -29,7 +29,7 @@
import org.springframework.stereotype.Controller;
import de.ids_mannheim.korap.web.utils.ResourceFilters;
-
+import de.ids_mannheim.korap.web.utils.SearchResourceFilters;
import de.ids_mannheim.korap.config.KustvaktConfiguration;
import de.ids_mannheim.korap.constant.OAuth2Scope;
import de.ids_mannheim.korap.exceptions.KustvaktException;
@@ -154,6 +154,7 @@
@POST
@Path("{version}/search")
@Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
+ @SearchResourceFilters
public Response searchPost (@Context SecurityContext context,
@Context Locale locale,
@Context HttpHeaders headers,
@@ -212,6 +213,7 @@
@GET
@Path("{version}/search")
@Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
+ @SearchResourceFilters
public Response searchGet (@Context SecurityContext securityContext,
@Context HttpServletRequest request,
@Context HttpHeaders headers, @Context Locale locale,
@@ -251,6 +253,7 @@
@GET
@Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
@Path("{version}/corpus/{corpusId}/{docId}/{textId}/{matchId}/matchInfo")
+ @SearchResourceFilters
public Response getMatchInfo (@Context SecurityContext ctx,
@Context HttpHeaders headers, @Context Locale locale,
@PathParam("corpusId") String corpusId,
@@ -271,6 +274,7 @@
@GET
@Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
@Path("{version}/corpus/{corpusId}/{docId}/{textId}/{matchId}")
+ @SearchResourceFilters
public Response retrieveMatchInfo (@Context SecurityContext ctx,
@Context HttpHeaders headers, @Context Locale locale,
@PathParam("corpusId") String corpusId,
@@ -286,37 +290,45 @@
// Highlights may also be a list of valid highlight classes
@QueryParam("hls") Boolean highlights) throws KustvaktException {
- Boolean expandToSentence = true;
- if (expansion != null && (expansion.equals("false") || expansion.equals("null"))) {
- expandToSentence = false;
+ TokenContext tokenContext = (TokenContext) ctx.getUserPrincipal();
+ try {
+ scopeService.verifyScope(tokenContext, OAuth2Scope.MATCH_INFO);
+ }
+ catch (KustvaktException e) {
+ throw kustvaktResponseHandler.throwit(e);
}
- TokenContext tokenContext = (TokenContext) ctx.getUserPrincipal();
- scopeService.verifyScope(tokenContext, OAuth2Scope.MATCH_INFO);
+ Boolean expandToSentence = true;
+ if (expansion != null
+ && (expansion.equals("false") || expansion.equals("null"))) {
+ expandToSentence = false;
+ }
spans = spans != null ? spans : false;
Boolean snippet = true;
Boolean tokens = false;
- if (snippetStr != null && (snippetStr.equals("false") || snippetStr.equals("null")))
+ if (snippetStr != null
+ && (snippetStr.equals("false") || snippetStr.equals("null")))
snippet = false;
- if (tokensStr != null && (tokensStr.equals("true") || tokensStr.equals("1") || tokensStr.equals("yes")))
+ if (tokensStr != null && (tokensStr.equals("true")
+ || tokensStr.equals("1") || tokensStr.equals("yes")))
tokens = true;
highlights = highlights != null ? highlights : false;
- if (layers == null || layers.isEmpty()) layers = new HashSet<>();
+ if (layers == null || layers.isEmpty())
+ layers = new HashSet<>();
- try{
- String results = searchService.retrieveMatchInfo(
- corpusId, docId,
- textId, matchId, true, foundries, tokenContext.getUsername(),
- headers, layers, spans, snippet, tokens,
- expandToSentence, highlights);
+ try {
+ String results = searchService.retrieveMatchInfo(corpusId, docId,
+ textId, matchId, true, foundries,
+ tokenContext.getUsername(), headers, layers, spans, snippet,
+ tokens, expandToSentence, highlights);
return Response.ok(results).build();
}
catch (KustvaktException e) {
throw kustvaktResponseHandler.throwit(e);
}
-
+
}
/*
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFilters.java b/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFilters.java
new file mode 100644
index 0000000..12dbf57
--- /dev/null
+++ b/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFilters.java
@@ -0,0 +1,22 @@
+package de.ids_mannheim.korap.web.utils;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Defines the list of {@link javax.ws.rs.container.ContainerRequestFilter}
+ * and {@link javax.ws.rs.container.ContainerResponseFilter}
+ * classes associated with a resource method.
+ * <p>
+ * This annotation can be specified on a class or on method(s). Specifying it
+ * at a class level means that it applies to all the methods in the class.
+ * Specifying it on a method means that it is applicable to that method only.
+ * If applied at both the class and methods level , the method value overrides
+ * the class value.
+ */
+@Target({ ElementType.TYPE, ElementType.METHOD })
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SearchResourceFilters {
+}
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFiltersFeature.java b/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFiltersFeature.java
new file mode 100644
index 0000000..45a7c4e
--- /dev/null
+++ b/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFiltersFeature.java
@@ -0,0 +1,58 @@
+package de.ids_mannheim.korap.web.utils;
+
+import java.util.List;
+
+import javax.ws.rs.container.DynamicFeature;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.FeatureContext;
+import javax.ws.rs.ext.Provider;
+
+import org.glassfish.jersey.model.internal.CommonConfig;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import de.ids_mannheim.korap.web.filter.APIVersionFilter;
+import edu.emory.mathcs.backport.java.util.Arrays;
+
+/**
+ * Registers {@link javax.ws.rs.container.ContainerRequestFilter}
+ * and {@link javax.ws.rs.container.ContainerResponseFilter}
+ * classes for a resource method annotated with {@link ResourceFilters}.
+ */
+@Provider
+@Component
+public class SearchResourceFiltersFeature implements DynamicFeature {
+
+ @Value("${search.resource.filters:AuthenticationFilter,DemoUserFilter}")
+ private String[] resourceFilters;
+
+ @Override
+ public void configure (ResourceInfo resourceInfo, FeatureContext context) {
+ SearchResourceFilters filters = resourceInfo.getResourceMethod()
+ .getAnnotation(SearchResourceFilters.class);
+ if (filters != null) {
+ CommonConfig con = (CommonConfig) context.getConfiguration();
+ con.getComponentBag().clear();
+ }
+ else {
+ filters = resourceInfo.getResourceClass()
+ .getAnnotation(SearchResourceFilters.class);
+ }
+
+ if (filters != null) {
+ List<?> list = Arrays.asList(resourceFilters);
+ if (!list.contains("APIVersionFilter")) {
+ context.register(APIVersionFilter.class);
+ }
+
+ for(String c : resourceFilters) {
+ try {
+ context.register(Class.forName("de.ids_mannheim.korap.web.filter." + c));
+ }
+ catch (ClassNotFoundException e) {
+ e.printStackTrace();
+ }
+ }
+ }
+ }
+}
diff --git a/full/Changes b/full/Changes
index 00c8159..fc593b0 100644
--- a/full/Changes
+++ b/full/Changes
@@ -3,6 +3,7 @@
- Support token array in matchinfo (fixes #570; diewald)
- Updated VC list API and deprecated owner VC list (addressed #580)
- Added user info web-service (solved #566)
+- Implemented configurable resource filters for search and match info APIs
# version 0.69.3
@@ -38,7 +39,9 @@
- Deprecate API token (JWT) web-service
- Fixed Slf4J binding
2023-02-28
-Exclude junit5 dependencies to keep test runner in Eclipse using JUnit 4
+- Exclude junit5 dependencies to keep test runner in Eclipse using JUnit 4
+2023-03-06
+- Fixed testing refresh token expiry.
# version 0.69.1
diff --git a/full/src/main/java/de/ids_mannheim/korap/oauth2/service/OAuth2ScopeServiceImpl.java b/full/src/main/java/de/ids_mannheim/korap/oauth2/service/OAuth2ScopeServiceImpl.java
index d2d59eb..9102ea3 100644
--- a/full/src/main/java/de/ids_mannheim/korap/oauth2/service/OAuth2ScopeServiceImpl.java
+++ b/full/src/main/java/de/ids_mannheim/korap/oauth2/service/OAuth2ScopeServiceImpl.java
@@ -108,6 +108,11 @@
@Override
public void verifyScope (TokenContext context, OAuth2Scope requiredScope)
throws KustvaktException {
+ if (context == null) {
+ throw new KustvaktException(StatusCodes.AUTHORIZATION_FAILED,
+ "Authentication required. Please log in!");
+ }
+
if (!adminDao.isAdmin(context.getUsername())
&& context.getTokenType().equals(TokenType.BEARER)) {
Map<String, Object> parameters = context.getParameters();
diff --git a/full/src/main/resources/kustvakt.conf b/full/src/main/resources/kustvakt.conf
index c410011..128ad0c 100644
--- a/full/src/main/resources/kustvakt.conf
+++ b/full/src/main/resources/kustvakt.conf
@@ -44,11 +44,25 @@
delete.group = soft
delete.group.member = soft
-# availability regex (only support | )
+
+# Availability regex only support |
+# It should be removed/commented when the data doesn't contain availability field.
+#
availability.regex.free = CC-BY.*
-availability.regex.public = ACA.*|QAO.NC
+availability.regex.public = ACA.*|QAO-NC
availability.regex.all = QAO.*
+
+# Define resource filters for search and match info API
+# AuthenticationFilter activates authentication using OAuth2 tokens
+# DemoUserFilter allows access to API without login
+#
+# Default values: AuthenticationFilter,DemoUserFilter
+#
+search.resource.filters=AuthenticationFilter,DemoUserFilter
+
+
+
# options referring to the security module!
# OAuth
@@ -81,6 +95,7 @@
security.secure.hash.algorithm=BCRYPT
security.encryption.loadFactor = 10
+# DEPRECATED
# JWT
security.jwt.issuer=korap.ids-mannheim.de
security.sharedSecret=this-is-shared-secret-code-for-JWT-Signing.It-must-contains-minimum-256-bits
@@ -94,8 +109,4 @@
security.idleTimeoutDuration = 25M
security.multipleLogIn = true
security.loginAttemptNum = 3
-security.authAttemptTTL = 45M
-
-#EM: deprecated and not used
-#security.validation.stringLength = 150
-#security.validation.emailLength = 50
+security.authAttemptTTL = 45M
\ No newline at end of file
diff --git a/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java b/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
index 0ce1c11..64f55bb 100644
--- a/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
@@ -1,12 +1,5 @@
package de.ids_mannheim.korap.config;
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.support.GenericApplicationContext;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-import org.springframework.web.context.support.GenericWebApplicationContext;
-
import org.glassfish.jersey.server.ResourceConfig;
import org.glassfish.jersey.servlet.ServletContainer;
import org.glassfish.jersey.test.DeploymentContext;
@@ -15,7 +8,16 @@
import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
import org.glassfish.jersey.test.spi.TestContainerException;
import org.glassfish.jersey.test.spi.TestContainerFactory;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.support.GenericApplicationContext;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.annotation.DirtiesContext.ClassMode;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.web.context.support.GenericWebApplicationContext;
+@DirtiesContext(classMode = ClassMode.BEFORE_CLASS)
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration("classpath:test-config.xml")
public abstract class SpringJerseyTest extends JerseyTest {
diff --git a/full/src/test/java/de/ids_mannheim/korap/scenario/ICCTest.java b/full/src/test/java/de/ids_mannheim/korap/scenario/ICCTest.java
new file mode 100644
index 0000000..25e3eb5
--- /dev/null
+++ b/full/src/test/java/de/ids_mannheim/korap/scenario/ICCTest.java
@@ -0,0 +1,188 @@
+package de.ids_mannheim.korap.scenario;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.junit.Test;
+import org.springframework.test.context.ContextConfiguration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import de.ids_mannheim.korap.authentication.http.HttpAuthorizationHandler;
+import de.ids_mannheim.korap.config.Attributes;
+import de.ids_mannheim.korap.config.SpringJerseyTest;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.exceptions.StatusCodes;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+/**
+ * <p>Test scenario for ICC (International Comparable Corpus)
+ * instance</p>
+ *
+ *
+ * The instance requires user authentication and access to data is
+ * restricted to only logged-in users.
+ *
+ * This class uses <em>test-config-icc.xml</em> spring XML config
+ * defining the location of a specific kustvakt configuration file for
+ * this instance:<em>kustvakt-icc.conf</em>. These files are not
+ * included in a Kustvakt jar file.
+ *
+ * When running a Kustvakt jar file, these files must be included in
+ * the classpath. In the example below, the files are placed together
+ * in the a folder named <em>config</em> and it is included in the
+ * classpath. Besides, <em>jdbc.properties</em> is required at the
+ * same folder as the jar.
+ *
+ * <p>
+ * <code>
+ * java -cp Kustvakt-full-0.69.3.jar:config
+ * de.ids_mannheim.korap.server.KustvaktServer --spring-config
+ * test-config-icc.xml
+ * </code>
+ * </p>
+ *
+ * <h1>Spring configuration file</h1>
+ * <p>
+ * For ICC, collectionRewrite in the Spring XML configuration must
+ * be disabled. This has been done in <em>test-config-icc.xml</em>.
+ * </p>
+ *
+ * <p>For testing, the ICC configuration uses HTTP Basic
+ * Authentication and doesn't use LDAP.</p>
+ *
+ * <p>For production, Basic Authentication must be
+ * disabled/commented.</p>
+ *
+ * <pre><code>
+ * <bean id="basic_auth"
+ * class="de.ids_mannheim.korap.authentication.BasicAuthentication"
+ * />
+ *
+ * <util:list id="kustvakt_authproviders"
+ * value-type="de.ids_mannheim.korap.interfaces.AuthenticationIface">
+ * <!-- <ref bean="basic_auth" /> -->
+ * </code>
+ * </pre>
+ *
+ * <h1>Kustvakt configuration file</h1>
+ * The configuration file: <em>kustvakt-icc.conf</em> includes the
+ * following setup:
+ *
+ * <ul>
+ * <li>
+ * <em>krill.indexDir</em> should indicate the location of the index.
+ * It is set to the wiki-index for the test.
+ * </li>
+ *
+ * <p>
+ * <code>krill.indexDir=../wiki-index</code>
+ * </p>
+ *
+ * <li>
+ * <em>availability.regex</em>
+ * properties should be removed or commented since the data doesn't
+ * contain availability and access to data is not determined by this
+ * field.
+ * </li>
+ *
+ * <li>
+ * Resource filter class names for the search and match info services
+ * should be defined by <em>search.resource.filters property</em>. For
+ * example, to restricts access with only authentication filter:</li>
+ *
+ * <p>
+ * <code>search.resource.filters=AuthenticationFilter </code>
+ * </p>
+ *
+ *
+ *
+ *
+ * <li><em>oauth2.password.authentication</em> indicating the
+ * authentication method to match usernames and password.
+ * <code>TEST</code> is a dummy authentication that doesn't do any
+ * matching. For production, it must be changed to
+ * <code>LDAP</code>.</li>
+ *
+ * <p><code>oauth2.password.authentication=LDAP</code></p>
+ *
+ * </ul>
+ *
+ * @author elma
+ * @see /src/main/resources/properties/jdbc.properties
+ */
+@ContextConfiguration("classpath:test-config-icc.xml")
+public class ICCTest extends SpringJerseyTest {
+
+ public final static String API_VERSION = "v1.0";
+ public String basicAuth;
+
+ public ICCTest () throws KustvaktException {
+ basicAuth = HttpAuthorizationHandler
+ .createBasicAuthorizationHeaderValue("user", "password");
+ }
+
+ @Test
+ public void searchWithoutLogin () throws KustvaktException {
+ Response r = target().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=das]").queryParam("ql", "poliqarp")
+ .request().get();
+
+ assertEquals(Status.UNAUTHORIZED.getStatusCode(), r.getStatus());
+
+ String entity = r.readEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+
+ assertEquals(StatusCodes.AUTHORIZATION_FAILED,
+ node.at("/errors/0/0").asInt());
+ }
+
+ @Test
+ public void searchWithLogin () throws KustvaktException {
+ Response r = target().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=das]").queryParam("ql", "poliqarp")
+ .request().header(Attributes.AUTHORIZATION, basicAuth).get();
+
+ assertEquals(Status.OK.getStatusCode(), r.getStatus());
+
+ String entity = r.readEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+ assertTrue(node.at("/matches").size() > 0);
+ }
+
+ @Test
+ public void matchInfoWithoutLogin () throws KustvaktException {
+ Response response = target().path(API_VERSION).path("corpus")
+ .path("WDD17").path("982").path("72848").path("p2815-2816")
+ .queryParam("foundry", "*").request().get();
+
+ assertEquals(Status.UNAUTHORIZED.getStatusCode(), response.getStatus());
+
+ String entity = response.readEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+
+ assertEquals(StatusCodes.AUTHORIZATION_FAILED,
+ node.at("/errors/0/0").asInt());
+ }
+
+ @Test
+ public void matchInfoWithLogin () throws KustvaktException {
+ Response response = target().path(API_VERSION).path("corpus")
+ .path("WDD17").path("982").path("72848").path("p2815-2816")
+ .queryParam("foundry", "*").request()
+ .header(Attributes.AUTHORIZATION, basicAuth).get();
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+ String entity = response.readEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+
+ assertTrue(node.at("/hasSnippet").asBoolean());
+ assertNotNull(node.at("/matchID").asText());
+ assertNotNull(node.at("/snippet").asText());
+ }
+}
diff --git a/full/src/test/resources/kustvakt-icc.conf b/full/src/test/resources/kustvakt-icc.conf
new file mode 100644
index 0000000..c029395
--- /dev/null
+++ b/full/src/test/resources/kustvakt-icc.conf
@@ -0,0 +1,105 @@
+# index dir
+krill.indexDir = ../wiki-index
+
+krill.index.commit.count = 134217000
+krill.index.commit.log = log/krill.commit.log
+krill.index.commit.auto = 500
+krill.index.relations.max = 100
+# Directory path of virtual corpora to cache
+krill.namedVC = vc
+krill.test = true
+
+# LDAP
+ldap.config = src/test/resources/test-ldap.conf
+
+# Kustvakt
+# multiple versions separated by space
+current.api.version = v1.0
+supported.api.version = v0.1 v1.0
+
+# server
+server.port=8089
+server.host=localhost
+
+# mail settings
+mail.enabled = false
+mail.receiver = test@localhost
+mail.sender = noreply@ids-mannheim.de
+mail.address.retrieval = test
+
+# mail.templates
+template.group.invitation = notification.vm
+
+# default foundries for specific layers
+default.foundry.partOfSpeech = tt
+default.foundry.lemma = tt
+default.foundry.orthography = opennlp
+default.foundry.dependency = malt
+default.foundry.constituent = corenlp
+default.foundry.morphology = marmot
+default.foundry.surface = base
+
+# delete configuration (default hard)
+# delete.auto.group = hard
+delete.group = soft
+delete.group.member = soft
+
+
+# availability regex only support |
+# It should be removed/commented when the data doesn't contain availability field.
+#
+# availability.regex.free = CC-BY.*
+# availability.regex.public = ACA.*|QAO-NC
+# availability.regex.all = QAO.*
+
+
+# Define resource filters for search and match info web-services
+#
+# AuthenticationFilter activates authentication using OAuth2 tokens
+# DemoUserFilter allows access to the services without login
+#
+# Default values: AuthenticationFilter,DemoUserFilter
+#
+search.resource.filters=AuthenticationFilter
+
+
+# options referring to the security module!
+
+# OAuth
+# (see de.ids_mannheim.korap.constant.AuthenticationMethod for possible
+# oauth.password.authentication values)
+oauth2.password.authentication = TEST
+oauth2.native.client.host = korap.ids-mannheim.de
+oauth2.max.attempts = 2
+# expiry in seconds (S), minutes (M), hours (H), days (D)
+oauth2.access.token.expiry = 3M
+oauth2.refresh.token.expiry = 90D
+oauth2.authorization.code.expiry = 10M
+# -- scopes separated by space
+oauth2.default.scopes = search match_info
+oauth2.client.credentials.scopes = client_info
+
+oauth2.initial.super.client=true
+
+# see SecureRandom Number Generation Algorithms
+# optional
+security.secure.random.algorithm=SHA1PRNG
+
+# see MessageDigest Algorithms
+# default MD5
+security.md.algoritm = SHA-256
+
+# secure hash support: BCRYPT
+security.secure.hash.algorithm=BCRYPT
+security.encryption.loadFactor = 10
+
+
+# DEPRECATED
+# JWT
+security.jwt.issuer=https://korap.ids-mannheim.de
+security.sharedSecret=this-is-shared-secret-code-for-JWT-Signing.It-must-contains-minimum-256-bits
+
+# token expiration time
+security.longTokenTTL = 1D
+security.tokenTTL = 2S
+security.shortTokenTTL = 1S
diff --git a/full/src/test/resources/kustvakt-test.conf b/full/src/test/resources/kustvakt-test.conf
index e72b1fd..430ad5f 100644
--- a/full/src/test/resources/kustvakt-test.conf
+++ b/full/src/test/resources/kustvakt-test.conf
@@ -1,36 +1,43 @@
-## index dir
+# Krill settings
+#
krill.indexDir = ../sample-index
krill.index.commit.count = 134217000
krill.index.commit.log = log/krill.commit.log
krill.index.commit.auto = 500
krill.index.relations.max = 100
-## Directory path of virtual corpora to cache
+# Directory path of virtual corpora to cache
krill.namedVC = vc
krill.test = true
-#LDAP
+# LDAP configuration file
+#
ldap.config = src/test/resources/test-ldap.conf
-# Kustvakt
+# Kustvakt versions
+#
# multiple versions separated by space
current.api.version = v1.0
supported.api.version = v0.1 v1.0
-## server
+# Server
+#
server.port=8089
server.host=localhost
-## mail settings
+# Mail settings
+#
mail.enabled = false
mail.receiver = test@localhost
mail.sender = noreply@ids-mannheim.de
mail.address.retrieval = test
-## mail.templates
+# Mail.templates
+#
template.group.invitation = notification.vm
-## default foundries for specific layers
+# Default foundries for specific layers (optional)
+#
default.foundry.partOfSpeech = tt
default.foundry.lemma = tt
default.foundry.orthography = opennlp
@@ -39,22 +46,35 @@
default.foundry.morphology = marmot
default.foundry.surface = base
-## delete configuration (default hard)
+# Delete configuration (default hard)
+#
# delete.auto.group = hard
delete.group = soft
delete.group.member = soft
-## availability regex
-## only support |
+# Availability regex only support |
+# It should be removed/commented when the data doesn't contain availability field.
+#
availability.regex.free = CC-BY.*
availability.regex.public = ACA.*|QAO-NC
availability.regex.all = QAO.*
-## options referring to the security module!
-## OAuth
-### (see de.ids_mannheim.korap.constant.AuthenticationMethod for possible
-### oauth.password.authentication values)
+# Define resource filters for search and match info API
+# AuthenticationFilter activates authentication using OAuth2 tokens
+# DemoUserFilter allows access to API without login
+#
+# Default values: AuthenticationFilter,DemoUserFilter
+#
+search.resource.filters=AuthenticationFilter,DemoUserFilter
+
+
+# options referring to the security module!
+
+# OAuth
+# (see de.ids_mannheim.korap.constant.AuthenticationMethod for possible
+# oauth.password.authentication values)
+#
oauth2.password.authentication = TEST
oauth2.native.client.host = korap.ids-mannheim.de
oauth2.max.attempts = 2
@@ -68,8 +88,8 @@
oauth2.initial.super.client=true
-## OpenId
-### multiple values are separated by space
+# OpenId
+# multiple values are separated by space
openid.grant.types = authorization_code
openid.response.types = code
openid.response.modes = query
@@ -86,29 +106,30 @@
#openid.term.of.service =
openid.service.doc = https://github.com/KorAP/Kustvakt/wiki
-## JWK
-## must be set for openid
+# JWK
+# must be set for openid
rsa.private = kustvakt_rsa.key
rsa.public = kustvakt_rsa_public.key
rsa.key.id = 74caa3a9-217c-49e6-94e9-2368fdd02c35
-## see SecureRandom Number Generation Algorithms
-## optional
+# see SecureRandom Number Generation Algorithms
+# optional
security.secure.random.algorithm=SHA1PRNG
-## see MessageDigest Algorithms
-## default MD5
+# see MessageDigest Algorithms
+# default MD5
security.md.algoritm = SHA-256
-### secure hash support: BCRYPT
+# secure hash support: BCRYPT
security.secure.hash.algorithm=BCRYPT
security.encryption.loadFactor = 10
-## JWT
+# DEPRECATED
+# JWT
security.jwt.issuer=https://korap.ids-mannheim.de
security.sharedSecret=this-is-shared-secret-code-for-JWT-Signing.It-must-contains-minimum-256-bits
-## token expiration time
+# token expiration time
security.longTokenTTL = 1D
security.tokenTTL = 2S
security.shortTokenTTL = 1S
@@ -118,7 +139,3 @@
security.multipleLogIn = true
security.loginAttemptNum = 3
security.authAttemptTTL = 45M
-
-#EM: deprecated and not used
-security.validation.stringLength = 150
-security.validation.emailLength = 50
diff --git a/full/src/test/resources/test-config-icc.xml b/full/src/test/resources/test-config-icc.xml
new file mode 100644
index 0000000..a98d1af
--- /dev/null
+++ b/full/src/test/resources/test-config-icc.xml
@@ -0,0 +1,358 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xmlns:p="http://www.springframework.org/schema/p" xmlns:util="http://www.springframework.org/schema/util"
+ xmlns:aop="http://www.springframework.org/schema/aop" xmlns:tx="http://www.springframework.org/schema/tx"
+ xmlns="http://www.springframework.org/schema/beans" xmlns:context="http://www.springframework.org/schema/context"
+ xmlns:cache="http://www.springframework.org/schema/cache"
+ xsi:schemaLocation="http://www.springframework.org/schema/beans
+ http://www.springframework.org/schema/beans/spring-beans.xsd
+ http://www.springframework.org/schema/tx
+ http://www.springframework.org/schema/tx/spring-tx.xsd
+ http://www.springframework.org/schema/aop
+ http://www.springframework.org/schema/aop/spring-aop.xsd
+ http://www.springframework.org/schema/context
+ http://www.springframework.org/schema/context/spring-context.xsd
+ http://www.springframework.org/schema/util
+ http://www.springframework.org/schema/util/spring-util.xsd">
+
+ <context:component-scan base-package="de.ids_mannheim.korap" />
+ <context:annotation-config />
+
+ <bean id="props"
+ class="org.springframework.beans.factory.config.PropertiesFactoryBean">
+ <property name="ignoreResourceNotFound" value="true" />
+ <property name="locations">
+ <array>
+ <value>file:./kustvakt-icc.conf</value>
+ <value>classpath:kustvakt-icc.conf</value>
+ </array>
+ </property>
+ </bean>
+
+ <bean id="placeholders"
+ class="org.springframework.context.support.PropertySourcesPlaceholderConfigurer">
+ <property name="ignoreResourceNotFound" value="true" />
+ <property name="locations">
+ <array>
+ <value>classpath:test-jdbc.properties</value>
+ <value>file:./jdbc.properties</value>
+ <value>classpath:properties/mail.properties</value>
+ <value>file:./mail.properties</value>
+ <value>classpath:hibernate.properties</value>
+ <value>file:./kustvakt-icc.conf</value>
+ <value>classpath:kustvakt-icc.conf</value>
+ </array>
+ </property>
+ </bean>
+
+ <bean id='cacheManager' class='org.springframework.cache.ehcache.EhCacheCacheManager'
+ p:cacheManager-ref='ehcache' />
+
+ <bean id='ehcache'
+ class='org.springframework.cache.ehcache.EhCacheManagerFactoryBean'
+ p:configLocation='classpath:ehcache.xml' p:shared='true' />
+
+ <bean id="dataSource" class="org.apache.commons.dbcp2.BasicDataSource"
+ lazy-init="true">
+ <!-- <property name="driverClassName" value="${jdbc.driverClassName}" /> -->
+ <property name="url" value="${jdbc.url}" />
+ <property name="username" value="${jdbc.username}" />
+ <property name="password" value="${jdbc.password}" />
+ <property name="maxTotal" value="4" />
+ <property name="maxIdle" value="1" />
+ <property name="minIdle" value="1" />
+ <property name="maxWaitMillis" value="15000" />
+ <!--<property name="poolPreparedStatements" value="true"/> -->
+ </bean>
+
+ <!-- use SingleConnection only for testing! -->
+ <bean id="sqliteDataSource"
+ class="org.springframework.jdbc.datasource.SingleConnectionDataSource"
+ lazy-init="true">
+ <!-- <property name="driverClassName" value="${jdbc.driverClassName}" /> -->
+ <property name="url" value="${jdbc.url}" />
+ <property name="username" value="${jdbc.username}" />
+ <property name="password" value="${jdbc.password}" />
+ <property name="connectionProperties">
+ <props>
+ <prop key="date_string_format">yyyy-MM-dd HH:mm:ss</prop>
+ </props>
+ </property>
+
+ <!-- Sqlite can only have a single connection -->
+ <property name="suppressClose">
+ <value>true</value>
+ </property>
+ </bean>
+
+ <bean id="c3p0DataSource" class="com.mchange.v2.c3p0.ComboPooledDataSource"
+ destroy-method="close">
+ <property name="driverClass" value="${jdbc.driverClassName}" />
+ <property name="jdbcUrl" value="${jdbc.url}" />
+ <property name="user" value="${jdbc.username}" />
+ <property name="password" value="${jdbc.password}" />
+ <property name="maxPoolSize" value="4" />
+ <property name="minPoolSize" value="1" />
+ <property name="maxStatements" value="1" />
+ <property name="testConnectionOnCheckout" value="true" />
+ </bean>
+
+ <!-- to configure database for sqlite, mysql, etc. migrations -->
+ <bean id="flywayConfig" class="org.flywaydb.core.api.configuration.ClassicConfiguration">
+ <!-- drop existing tables and create new tables -->
+ <property name="validateOnMigrate" value="true" />
+ <property name="cleanOnValidationError" value="true" />
+ <property name="baselineOnMigrate" value="false" />
+ <property name="locations" value="#{'${jdbc.schemaPath}'.split(',')}"/>
+ <property name="dataSource" ref="sqliteDataSource" />
+ <!-- <property name="dataSource" ref="dataSource" /> -->
+ <property name="outOfOrder" value="true" />
+ </bean>
+
+ <bean id="flyway" class="org.flywaydb.core.Flyway" init-method="migrate">
+ <constructor-arg ref="flywayConfig"/>
+ </bean>
+
+
+ <bean id="kustvakt_db" class="de.ids_mannheim.korap.handlers.JDBCClient">
+ <!-- <constructor-arg index="0" ref="dataSource" /> -->
+ <constructor-arg index="0" ref="sqliteDataSource" />
+ <property name="database" value="${jdbc.database}" />
+ </bean>
+
+ <bean id="entityManagerFactory"
+ class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
+ <!-- <property name="dataSource" ref="dataSource" /> -->
+ <property name="dataSource" ref="sqliteDataSource" />
+ <property name="packagesToScan">
+ <array>
+ <value>de.ids_mannheim.korap.entity</value>
+ <value>de.ids_mannheim.korap.oauth2.entity</value>
+ </array>
+ </property>
+ <property name="jpaVendorAdapter">
+ <bean id="jpaVendorAdapter"
+ class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
+ <property name="databasePlatform" value="${hibernate.dialect}" />
+ </bean>
+ </property>
+ <property name="jpaProperties">
+ <props>
+ <prop key="hibernate.dialect">${hibernate.dialect}</prop>
+ <prop key="hibernate.hbm2ddl.auto">${hibernate.hbm2ddl.auto}</prop>
+ <prop key="hibernate.show_sql">${hibernate.show_sql}</prop>
+ <prop key="hibernate.cache.use_query_cache">${hibernate.cache.use_query_cache}</prop>
+ <prop key="hibernate.cache.use_second_level_cache">${hibernate.cache.use_second_level_cache}
+ </prop>
+ <prop key="hibernate.cache.provider_class">${hibernate.cache.provider}</prop>
+ <prop key="hibernate.cache.region.factory_class">${hibernate.cache.region.factory}</prop>
+ <prop key="hibernate.jdbc.time_zone">${hibernate.jdbc.time_zone}</prop>
+ <!-- <prop key="net.sf.ehcache.configurationResourceName">classpath:ehcache.xml</prop> -->
+ </props>
+ </property>
+ </bean>
+
+ <tx:annotation-driven proxy-target-class="true"
+ transaction-manager="transactionManager" />
+ <bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
+ <property name="entityManagerFactory" ref="entityManagerFactory" />
+ </bean>
+
+ <bean id="transactionTemplate"
+ class="org.springframework.transaction.support.TransactionTemplate">
+ <constructor-arg ref="transactionManager" />
+ </bean>
+
+ <!-- Data access objects -->
+ <bean id="adminDao" class="de.ids_mannheim.korap.dao.AdminDaoImpl" />
+ <bean id="resourceDao" class="de.ids_mannheim.korap.dao.ResourceDao" />
+ <bean id="accessScopeDao" class="de.ids_mannheim.korap.oauth2.dao.AccessScopeDao" />
+ <bean id="authorizationDao" class="de.ids_mannheim.korap.oauth2.dao.CachedAuthorizationDaoImpl" />
+
+ <!-- Services -->
+ <bean id="scopeService" class="de.ids_mannheim.korap.oauth2.service.OAuth2ScopeServiceImpl" />
+
+
+ <!-- Controller -->
+
+
+ <!-- props are injected from default-config.xml -->
+ <bean id="kustvakt_config" class="de.ids_mannheim.korap.config.FullConfiguration">
+ <constructor-arg name="properties" ref="props" />
+ </bean>
+
+ <bean id="initializator" class="de.ids_mannheim.de.init.Initializator"
+ init-method="initTest">
+ </bean>
+
+ <!-- Krill -->
+ <bean id="search_krill" class="de.ids_mannheim.korap.web.SearchKrill">
+ <constructor-arg value="${krill.indexDir}" />
+ </bean>
+
+ <!-- Validator -->
+ <bean id="validator" class="de.ids_mannheim.korap.validator.ApacheValidator"/>
+
+ <!-- URLValidator -->
+ <bean id="redirectURIValidator" class="org.apache.commons.validator.routines.UrlValidator">
+ <constructor-arg value="http,https" index="0" />
+ <constructor-arg index="1" type="long"
+ value="#{T(org.apache.commons.validator.routines.UrlValidator).ALLOW_LOCAL_URLS +
+ T(org.apache.commons.validator.routines.UrlValidator).NO_FRAGMENTS}"/>
+ </bean>
+ <bean id="urlValidator" class="org.apache.commons.validator.routines.UrlValidator">
+ <constructor-arg value="http,https" />
+ </bean>
+
+ <!-- Rewrite -->
+ <bean id="foundryRewrite" class="de.ids_mannheim.korap.rewrite.FoundryRewrite"/>
+ <bean id="collectionRewrite" class="de.ids_mannheim.korap.rewrite.CollectionRewrite"/>
+ <bean id="collectionCleanRewrite" class="de.ids_mannheim.korap.rewrite.CollectionCleanRewrite"/>
+ <bean id="virtualCorpusRewrite" class="de.ids_mannheim.korap.rewrite.VirtualCorpusRewrite"/>
+ <bean id="collectionConstraint" class="de.ids_mannheim.korap.rewrite.CollectionConstraint"/>
+ <bean id="queryReferenceRewrite" class="de.ids_mannheim.korap.rewrite.QueryReferenceRewrite"/>
+
+ <util:list id="rewriteTasks"
+ value-type="de.ids_mannheim.korap.rewrite.RewriteTask">
+ <!-- <ref bean="collectionConstraint" />
+ <ref bean="collectionCleanRewrite" /> -->
+ <ref bean="foundryRewrite" />
+ <!-- <ref bean="collectionRewrite" /> -->
+ <ref bean="virtualCorpusRewrite" />
+ <ref bean="queryReferenceRewrite" />
+ </util:list>
+
+ <bean id="rewriteHandler" class="de.ids_mannheim.korap.rewrite.RewriteHandler">
+ <constructor-arg ref="rewriteTasks"/>
+ </bean>
+
+ <bean id="kustvakt_auditing" class="de.ids_mannheim.korap.handlers.JDBCAuditing">
+ <constructor-arg ref="kustvakt_db" />
+ </bean>
+
+ <bean id="kustvaktResponseHandler" class="de.ids_mannheim.korap.web.KustvaktResponseHandler">
+ <constructor-arg index="0" name="iface" ref="kustvakt_auditing" />
+ </bean>
+
+ <!-- OAuth -->
+ <bean id="oauth2ResponseHandler" class="de.ids_mannheim.korap.web.OAuth2ResponseHandler">
+ <constructor-arg index="0" name="iface" ref="kustvakt_auditing" />
+ </bean>
+
+ <bean id="mdGenerator" class="org.apache.oltu.oauth2.as.issuer.MD5Generator">
+ </bean>
+ <bean id="oauthIssuer" class="org.apache.oltu.oauth2.as.issuer.OAuthIssuerImpl">
+ <constructor-arg index="0" ref="mdGenerator" />
+ </bean>
+
+ <bean id="kustvakt_userdb" class="de.ids_mannheim.korap.handlers.EntityDao">
+ <constructor-arg ref="kustvakt_db" />
+ </bean>
+
+ <bean name="kustvakt_encryption" class="de.ids_mannheim.korap.encryption.KustvaktEncryption">
+ <constructor-arg ref="kustvakt_config" />
+ </bean>
+
+ <!-- authentication providers to use -->
+ <bean id="basic_auth"
+ class="de.ids_mannheim.korap.authentication.BasicAuthentication" />
+
+ <bean id="oauth2_auth"
+ class="de.ids_mannheim.korap.authentication.OAuth2Authentication" />
+
+
+ <util:list id="kustvakt_authproviders"
+ value-type="de.ids_mannheim.korap.interfaces.AuthenticationIface">
+ <ref bean="basic_auth" />
+ <ref bean="oauth2_auth" />
+ </util:list>
+
+
+ <bean id="userdata_details" class="de.ids_mannheim.korap.handlers.UserDetailsDao">
+ <constructor-arg ref="kustvakt_db" />
+ </bean>
+
+ <bean id="userdata_settings" class="de.ids_mannheim.korap.handlers.UserSettingsDao">
+ <constructor-arg ref="kustvakt_db" />
+ </bean>
+
+ <util:list id="kustvakt_userdata"
+ value-type="de.ids_mannheim.korap.interfaces.db.UserdataDbIface">
+ <ref bean="userdata_details" />
+ <ref bean="userdata_settings" />
+ </util:list>
+
+ <!-- specify type for constructor argument -->
+ <bean id="authenticationManager"
+ class="de.ids_mannheim.korap.authentication.KustvaktAuthenticationManager">
+ <constructor-arg type="de.ids_mannheim.korap.interfaces.EntityHandlerIface"
+ ref="kustvakt_userdb" />
+ <constructor-arg type="de.ids_mannheim.korap.interfaces.EncryptionIface"
+ ref="kustvakt_encryption" />
+ <constructor-arg ref="kustvakt_config" />
+ <constructor-arg type="de.ids_mannheim.korap.interfaces.db.AuditingIface"
+ ref="kustvakt_auditing" />
+ <constructor-arg ref="kustvakt_userdata" />
+ <!-- inject authentication providers to use -->
+ <property name="providers" ref="kustvakt_authproviders" />
+ </bean>
+
+ <!-- todo: if db interfaces not loaded via spring, does transaction even
+ work then? -->
+ <!-- the transactional advice (i.e. what 'happens'; see the <aop:advisor/>
+ bean below) -->
+ <tx:advice id="txAdvice" transaction-manager="txManager">
+ <!-- the transactional semantics... -->
+ <tx:attributes>
+ <!-- all methods starting with 'get' are read-only -->
+ <tx:method name="get*" read-only="true" rollback-for="KorAPException" />
+ <!-- other methods use the default transaction settings (see below) -->
+ <tx:method name="*" rollback-for="KorAPException" />
+ </tx:attributes>
+ </tx:advice>
+
+ <!-- ensure that the above transactional advice runs for any execution of
+ an operation defined by the service interface -->
+ <aop:config>
+ <aop:pointcut id="service"
+ expression="execution(* de.ids_mannheim.korap.interfaces.db.*.*(..))" />
+ <aop:advisor advice-ref="txAdvice" pointcut-ref="service" />
+ </aop:config>
+
+ <!-- similarly, don't forget the PlatformTransactionManager -->
+ <bean id="txManager"
+ class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
+ <property name="dataSource" ref="dataSource" />
+ </bean>
+
+ <!-- mail -->
+ <bean id="authenticator" class="de.ids_mannheim.korap.service.MailAuthenticator">
+ <constructor-arg index="0" value="${mail.username}" />
+ <constructor-arg index="1" value="${mail.password}" />
+ </bean>
+ <bean id="smtpSession" class="javax.mail.Session" factory-method="getInstance">
+ <constructor-arg index="0">
+ <props>
+ <prop key="mail.smtp.submitter">${mail.username}</prop>
+ <prop key="mail.smtp.auth">${mail.auth}</prop>
+ <prop key="mail.smtp.host">${mail.host}</prop>
+ <prop key="mail.smtp.port">${mail.port}</prop>
+ <prop key="mail.smtp.starttls.enable">${mail.starttls.enable}</prop>
+ <prop key="mail.smtp.connectiontimeout">${mail.connectiontimeout}</prop>
+ </props>
+ </constructor-arg>
+ <constructor-arg index="1" ref="authenticator" />
+ </bean>
+ <bean id="mailSender" class="org.springframework.mail.javamail.JavaMailSenderImpl">
+ <property name="session" ref="smtpSession" />
+ </bean>
+ <bean id="velocityEngine" class="org.apache.velocity.app.VelocityEngine">
+ <constructor-arg index="0">
+ <props>
+ <prop key="resource.loader">class</prop>
+ <prop key="class.resource.loader.class">org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader
+ </prop>
+ </props>
+ </constructor-arg>
+ </bean>
+</beans>
diff --git a/full/src/test/resources/test-config.xml b/full/src/test/resources/test-config.xml
index aaea39c..cffc34e 100644
--- a/full/src/test/resources/test-config.xml
+++ b/full/src/test/resources/test-config.xml
@@ -39,6 +39,7 @@
<value>classpath:properties/mail.properties</value>
<value>file:./mail.properties</value>
<value>classpath:test-hibernate.properties</value>
+ <value>file:./kustvakt-test.conf</value>
<value>classpath:kustvakt-test.conf</value>
</array>
</property>
diff --git a/wiki-index/_0.cfe b/wiki-index/_0.cfe
new file mode 100644
index 0000000..17325ac
--- /dev/null
+++ b/wiki-index/_0.cfe
Binary files differ
diff --git a/wiki-index/_0.cfs b/wiki-index/_0.cfs
new file mode 100644
index 0000000..65b7ade
--- /dev/null
+++ b/wiki-index/_0.cfs
Binary files differ
diff --git a/wiki-index/_0.si b/wiki-index/_0.si
new file mode 100644
index 0000000..1d5b267
--- /dev/null
+++ b/wiki-index/_0.si
Binary files differ
diff --git a/wiki-index/segments_1 b/wiki-index/segments_1
new file mode 100644
index 0000000..4df408b
--- /dev/null
+++ b/wiki-index/segments_1
Binary files differ
diff --git a/wiki-index/write.lock b/wiki-index/write.lock
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/wiki-index/write.lock