Implemented configurable resource filters for search and match info APIs

Added tests and settings for ICC scenario.

Change-Id: If22b96cd12f2a39c134a45f9e3e4b2da8bcd36dc
diff --git a/.gitignore b/.gitignore
index 1e8daea..1147d0b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -17,6 +17,8 @@
 /lite/krill_cache/
 **/ldap.conf
 
+icc-index
+
 # FB specific Kustvakt.conf
 src\main\resources\Kustvakt.conf
 /temp/
diff --git a/core/Changes b/core/Changes
index cd92fca..3590c21 100644
--- a/core/Changes
+++ b/core/Changes
@@ -2,6 +2,7 @@
 
 - Support token array in matchinfo (fixes #570; diewald)
 - Added user info web-service (solved #566)
+- Implemented configurable resource filters for search and match info APIs
 
 # version 0.69.3
 
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java b/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
index 3cc768b..7c180e8 100644
--- a/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
+++ b/core/src/main/java/de/ids_mannheim/korap/web/controller/SearchController.java
@@ -29,7 +29,7 @@
 import org.springframework.stereotype.Controller;
 
 import de.ids_mannheim.korap.web.utils.ResourceFilters;
-
+import de.ids_mannheim.korap.web.utils.SearchResourceFilters;
 import de.ids_mannheim.korap.config.KustvaktConfiguration;
 import de.ids_mannheim.korap.constant.OAuth2Scope;
 import de.ids_mannheim.korap.exceptions.KustvaktException;
@@ -154,6 +154,7 @@
     @POST
     @Path("{version}/search")
     @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
+    @SearchResourceFilters
     public Response searchPost (@Context SecurityContext context,
             @Context Locale locale, 
             @Context HttpHeaders headers,
@@ -212,6 +213,7 @@
     @GET
     @Path("{version}/search")
     @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
+    @SearchResourceFilters
     public Response searchGet (@Context SecurityContext securityContext,
             @Context HttpServletRequest request,
             @Context HttpHeaders headers, @Context Locale locale,
@@ -251,6 +253,7 @@
     @GET
     @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
     @Path("{version}/corpus/{corpusId}/{docId}/{textId}/{matchId}/matchInfo")
+    @SearchResourceFilters
     public Response getMatchInfo (@Context SecurityContext ctx,
             @Context HttpHeaders headers, @Context Locale locale,
             @PathParam("corpusId") String corpusId,
@@ -271,6 +274,7 @@
     @GET
     @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
     @Path("{version}/corpus/{corpusId}/{docId}/{textId}/{matchId}")
+    @SearchResourceFilters
     public Response retrieveMatchInfo (@Context SecurityContext ctx,
             @Context HttpHeaders headers, @Context Locale locale,
             @PathParam("corpusId") String corpusId,
@@ -286,37 +290,45 @@
             // Highlights may also be a list of valid highlight classes
             @QueryParam("hls") Boolean highlights) throws KustvaktException {
 
-        Boolean expandToSentence = true;
-        if (expansion != null && (expansion.equals("false") || expansion.equals("null"))) {
-            expandToSentence = false;
+        TokenContext tokenContext = (TokenContext) ctx.getUserPrincipal();
+        try {
+            scopeService.verifyScope(tokenContext, OAuth2Scope.MATCH_INFO);
+        }
+        catch (KustvaktException e) {
+            throw kustvaktResponseHandler.throwit(e);
         }
 
-        TokenContext tokenContext = (TokenContext) ctx.getUserPrincipal();
-        scopeService.verifyScope(tokenContext, OAuth2Scope.MATCH_INFO);
+        Boolean expandToSentence = true;
+        if (expansion != null
+                && (expansion.equals("false") || expansion.equals("null"))) {
+            expandToSentence = false;
+        }
         spans = spans != null ? spans : false;
         Boolean snippet = true;
         Boolean tokens = false;
-        if (snippetStr != null && (snippetStr.equals("false") || snippetStr.equals("null")))
+        if (snippetStr != null
+                && (snippetStr.equals("false") || snippetStr.equals("null")))
             snippet = false;
 
-        if (tokensStr != null && (tokensStr.equals("true") || tokensStr.equals("1") || tokensStr.equals("yes")))
+        if (tokensStr != null && (tokensStr.equals("true")
+                || tokensStr.equals("1") || tokensStr.equals("yes")))
             tokens = true;
 
         highlights = highlights != null ? highlights : false;
-        if (layers == null || layers.isEmpty()) layers = new HashSet<>();
+        if (layers == null || layers.isEmpty())
+            layers = new HashSet<>();
 
-        try{
-            String results = searchService.retrieveMatchInfo(
-                corpusId, docId,
-                textId, matchId, true, foundries, tokenContext.getUsername(),
-                headers, layers, spans, snippet, tokens,
-                expandToSentence, highlights);
+        try {
+            String results = searchService.retrieveMatchInfo(corpusId, docId,
+                    textId, matchId, true, foundries,
+                    tokenContext.getUsername(), headers, layers, spans, snippet,
+                    tokens, expandToSentence, highlights);
             return Response.ok(results).build();
         }
         catch (KustvaktException e) {
             throw kustvaktResponseHandler.throwit(e);
         }
-        
+
     }
 
     /*
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFilters.java b/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFilters.java
new file mode 100644
index 0000000..12dbf57
--- /dev/null
+++ b/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFilters.java
@@ -0,0 +1,22 @@
+package de.ids_mannheim.korap.web.utils;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * Defines the list of {@link javax.ws.rs.container.ContainerRequestFilter}
+ * and {@link javax.ws.rs.container.ContainerResponseFilter}
+ * classes associated with a resource method.
+ * <p>
+ * This annotation can be specified on a class or on method(s). Specifying it
+ * at a class level means that it applies to all the methods in the class.
+ * Specifying it on a method means that it is applicable to that method only.
+ * If applied at both the class and methods level , the method value overrides
+ * the class value.
+ */
+@Target({ ElementType.TYPE, ElementType.METHOD })
+@Retention(RetentionPolicy.RUNTIME)
+public @interface SearchResourceFilters {
+}
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFiltersFeature.java b/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFiltersFeature.java
new file mode 100644
index 0000000..45a7c4e
--- /dev/null
+++ b/core/src/main/java/de/ids_mannheim/korap/web/utils/SearchResourceFiltersFeature.java
@@ -0,0 +1,58 @@
+package de.ids_mannheim.korap.web.utils;
+
+import java.util.List;
+
+import javax.ws.rs.container.DynamicFeature;
+import javax.ws.rs.container.ResourceInfo;
+import javax.ws.rs.core.FeatureContext;
+import javax.ws.rs.ext.Provider;
+
+import org.glassfish.jersey.model.internal.CommonConfig;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import de.ids_mannheim.korap.web.filter.APIVersionFilter;
+import edu.emory.mathcs.backport.java.util.Arrays;
+
+/**
+ * Registers {@link javax.ws.rs.container.ContainerRequestFilter}
+ * and {@link javax.ws.rs.container.ContainerResponseFilter}
+ * classes for a resource method annotated with {@link ResourceFilters}.
+ */
+@Provider
+@Component
+public class SearchResourceFiltersFeature implements DynamicFeature {
+
+    @Value("${search.resource.filters:AuthenticationFilter,DemoUserFilter}")
+    private String[] resourceFilters;
+    
+    @Override
+    public void configure (ResourceInfo resourceInfo, FeatureContext context) {
+        SearchResourceFilters filters = resourceInfo.getResourceMethod()
+                .getAnnotation(SearchResourceFilters.class);
+        if (filters != null) {
+            CommonConfig con = (CommonConfig) context.getConfiguration();
+            con.getComponentBag().clear();
+        }        
+        else {
+            filters = resourceInfo.getResourceClass()
+            .getAnnotation(SearchResourceFilters.class);
+        }
+        
+        if (filters != null) {
+            List<?> list = Arrays.asList(resourceFilters);
+            if (!list.contains("APIVersionFilter")) {
+                context.register(APIVersionFilter.class);
+            }
+            
+             for(String c : resourceFilters) {
+                    try {
+                        context.register(Class.forName("de.ids_mannheim.korap.web.filter." + c));
+                    }
+                    catch (ClassNotFoundException e) {
+                        e.printStackTrace();
+                    }
+            }
+        }
+    }
+}
diff --git a/full/Changes b/full/Changes
index 00c8159..fc593b0 100644
--- a/full/Changes
+++ b/full/Changes
@@ -3,6 +3,7 @@
 - Support token array in matchinfo (fixes #570; diewald)
 - Updated VC list API and deprecated owner VC list (addressed #580)
 - Added user info web-service (solved #566)
+- Implemented configurable resource filters for search and match info APIs
 
 # version 0.69.3
 
@@ -38,7 +39,9 @@
 - Deprecate API token (JWT) web-service
 - Fixed Slf4J binding
 2023-02-28
-Exclude junit5 dependencies to keep test runner in Eclipse using JUnit 4
+- Exclude junit5 dependencies to keep test runner in Eclipse using JUnit 4
+2023-03-06
+- Fixed testing refresh token expiry.
 
 
 # version 0.69.1
diff --git a/full/src/main/java/de/ids_mannheim/korap/oauth2/service/OAuth2ScopeServiceImpl.java b/full/src/main/java/de/ids_mannheim/korap/oauth2/service/OAuth2ScopeServiceImpl.java
index d2d59eb..9102ea3 100644
--- a/full/src/main/java/de/ids_mannheim/korap/oauth2/service/OAuth2ScopeServiceImpl.java
+++ b/full/src/main/java/de/ids_mannheim/korap/oauth2/service/OAuth2ScopeServiceImpl.java
@@ -108,6 +108,11 @@
     @Override
     public void verifyScope (TokenContext context, OAuth2Scope requiredScope)
             throws KustvaktException {
+        if (context == null) {
+            throw new KustvaktException(StatusCodes.AUTHORIZATION_FAILED,
+                    "Authentication required. Please log in!");
+        }
+        
         if (!adminDao.isAdmin(context.getUsername())
                 && context.getTokenType().equals(TokenType.BEARER)) {
             Map<String, Object> parameters = context.getParameters();
diff --git a/full/src/main/resources/kustvakt.conf b/full/src/main/resources/kustvakt.conf
index c410011..128ad0c 100644
--- a/full/src/main/resources/kustvakt.conf
+++ b/full/src/main/resources/kustvakt.conf
@@ -44,11 +44,25 @@
 delete.group = soft
 delete.group.member = soft
 
-# availability regex (only support | )
+
+# Availability regex only support |
+# It should be removed/commented when the data doesn't contain availability field.
+# 
 availability.regex.free = CC-BY.*
-availability.regex.public = ACA.*|QAO.NC
+availability.regex.public = ACA.*|QAO-NC
 availability.regex.all = QAO.*
 
+
+# Define resource filters for search and match info API
+# AuthenticationFilter activates authentication using OAuth2 tokens
+# DemoUserFilter allows access to API without login
+# 
+# Default values: AuthenticationFilter,DemoUserFilter
+#
+search.resource.filters=AuthenticationFilter,DemoUserFilter
+
+
+
 # options referring to the security module!
 
 # OAuth 
@@ -81,6 +95,7 @@
 security.secure.hash.algorithm=BCRYPT
 security.encryption.loadFactor = 10
 
+# DEPRECATED
 # JWT
 security.jwt.issuer=korap.ids-mannheim.de
 security.sharedSecret=this-is-shared-secret-code-for-JWT-Signing.It-must-contains-minimum-256-bits
@@ -94,8 +109,4 @@
 security.idleTimeoutDuration = 25M
 security.multipleLogIn = true
 security.loginAttemptNum = 3
-security.authAttemptTTL = 45M
-
-#EM: deprecated and not used
-#security.validation.stringLength = 150
-#security.validation.emailLength = 50
+security.authAttemptTTL = 45M
\ No newline at end of file
diff --git a/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java b/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
index 0ce1c11..64f55bb 100644
--- a/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/config/SpringJerseyTest.java
@@ -1,12 +1,5 @@
 package de.ids_mannheim.korap.config;
 
-import org.junit.runner.RunWith;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.context.support.GenericApplicationContext;
-import org.springframework.test.context.ContextConfiguration;
-import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
-import org.springframework.web.context.support.GenericWebApplicationContext;
-
 import org.glassfish.jersey.server.ResourceConfig;
 import org.glassfish.jersey.servlet.ServletContainer;
 import org.glassfish.jersey.test.DeploymentContext;
@@ -15,7 +8,16 @@
 import org.glassfish.jersey.test.grizzly.GrizzlyWebTestContainerFactory;
 import org.glassfish.jersey.test.spi.TestContainerException;
 import org.glassfish.jersey.test.spi.TestContainerFactory;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.support.GenericApplicationContext;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.annotation.DirtiesContext.ClassMode;
+import org.springframework.test.context.ContextConfiguration;
+import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
+import org.springframework.web.context.support.GenericWebApplicationContext;
 
+@DirtiesContext(classMode = ClassMode.BEFORE_CLASS)
 @RunWith(SpringJUnit4ClassRunner.class)
 @ContextConfiguration("classpath:test-config.xml")
 public abstract class SpringJerseyTest extends JerseyTest {
diff --git a/full/src/test/java/de/ids_mannheim/korap/scenario/ICCTest.java b/full/src/test/java/de/ids_mannheim/korap/scenario/ICCTest.java
new file mode 100644
index 0000000..25e3eb5
--- /dev/null
+++ b/full/src/test/java/de/ids_mannheim/korap/scenario/ICCTest.java
@@ -0,0 +1,188 @@
+package de.ids_mannheim.korap.scenario;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
+import org.junit.Test;
+import org.springframework.test.context.ContextConfiguration;
+
+import com.fasterxml.jackson.databind.JsonNode;
+
+import de.ids_mannheim.korap.authentication.http.HttpAuthorizationHandler;
+import de.ids_mannheim.korap.config.Attributes;
+import de.ids_mannheim.korap.config.SpringJerseyTest;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.exceptions.StatusCodes;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+/**
+ * <p>Test scenario for ICC (International Comparable Corpus)
+ * instance</p>
+ * 
+ * 
+ * The instance requires user authentication and access to data is
+ * restricted to only logged-in users.
+ * 
+ * This class uses <em>test-config-icc.xml</em> spring XML config
+ * defining the location of a specific kustvakt configuration file for
+ * this instance:<em>kustvakt-icc.conf</em>. These files are not
+ * included in a Kustvakt jar file.
+ * 
+ * When running a Kustvakt jar file, these files must be included in
+ * the classpath. In the example below, the files are placed together
+ * in the a folder named <em>config</em> and it is included in the
+ * classpath. Besides, <em>jdbc.properties</em> is required at the
+ * same folder as the jar.
+ * 
+ * <p>
+ * <code>
+ * java -cp Kustvakt-full-0.69.3.jar:config
+ * de.ids_mannheim.korap.server.KustvaktServer --spring-config
+ * test-config-icc.xml
+ * </code>
+ * </p>
+ * 
+ * <h1>Spring configuration file</h1>
+ * <p>
+ * For ICC, collectionRewrite in the Spring XML configuration must
+ * be disabled. This has been done in <em>test-config-icc.xml</em>.
+ * </p>
+ * 
+ * <p>For testing, the ICC configuration uses HTTP Basic
+ * Authentication and doesn't use LDAP.</p>
+ * 
+ * <p>For production, Basic Authentication must be
+ * disabled/commented.</p>
+ * 
+ * <pre><code>
+ * &lt;bean id="basic_auth"
+ * class="de.ids_mannheim.korap.authentication.BasicAuthentication"
+ * /&gt;
+ * 
+ * &lt;util:list id="kustvakt_authproviders"
+ * value-type="de.ids_mannheim.korap.interfaces.AuthenticationIface"&gt;
+ * &lt;!-- &lt;ref bean="basic_auth" /&gt; --&gt;
+ * </code>
+ * </pre>
+ * 
+ * <h1>Kustvakt configuration file</h1>
+ * The configuration file: <em>kustvakt-icc.conf</em> includes the
+ * following setup:
+ * 
+ * <ul>
+ * <li>
+ * <em>krill.indexDir</em> should indicate the location of the index.
+ * It is set to the wiki-index for the test.
+ * </li>
+ * 
+ * <p>
+ * <code>krill.indexDir=../wiki-index</code>
+ * </p>
+ * 
+ * <li>
+ * <em>availability.regex</em>
+ * properties should be removed or commented since the data doesn't
+ * contain availability and access to data is not determined by this
+ * field.
+ * </li>
+ * 
+ * <li>
+ * Resource filter class names for the search and match info services
+ * should be defined by <em>search.resource.filters property</em>. For
+ * example, to restricts access with only authentication filter:</li>
+ * 
+ * <p>
+ * <code>search.resource.filters=AuthenticationFilter </code>
+ * </p>
+ * 
+ * 
+ * 
+ * 
+ * <li><em>oauth2.password.authentication</em> indicating the
+ * authentication method to match usernames and password.
+ * <code>TEST</code> is a dummy authentication that doesn't do any
+ * matching. For production, it must be changed to
+ * <code>LDAP</code>.</li>
+ * 
+ * <p><code>oauth2.password.authentication=LDAP</code></p>
+ * 
+ * </ul>
+ * 
+ * @author elma
+ * @see /src/main/resources/properties/jdbc.properties
+ */
+@ContextConfiguration("classpath:test-config-icc.xml")
+public class ICCTest extends SpringJerseyTest {
+
+    public final static String API_VERSION = "v1.0";
+    public String basicAuth;
+
+    public ICCTest () throws KustvaktException {
+        basicAuth = HttpAuthorizationHandler
+                .createBasicAuthorizationHeaderValue("user", "password");
+    }
+
+    @Test
+    public void searchWithoutLogin () throws KustvaktException {
+        Response r = target().path(API_VERSION).path("search")
+                .queryParam("q", "[orth=das]").queryParam("ql", "poliqarp")
+                .request().get();
+
+        assertEquals(Status.UNAUTHORIZED.getStatusCode(), r.getStatus());
+
+        String entity = r.readEntity(String.class);
+        JsonNode node = JsonUtils.readTree(entity);
+
+        assertEquals(StatusCodes.AUTHORIZATION_FAILED,
+                node.at("/errors/0/0").asInt());
+    }
+
+    @Test
+    public void searchWithLogin () throws KustvaktException {
+        Response r = target().path(API_VERSION).path("search")
+                .queryParam("q", "[orth=das]").queryParam("ql", "poliqarp")
+                .request().header(Attributes.AUTHORIZATION, basicAuth).get();
+
+        assertEquals(Status.OK.getStatusCode(), r.getStatus());
+
+        String entity = r.readEntity(String.class);
+        JsonNode node = JsonUtils.readTree(entity);
+        assertTrue(node.at("/matches").size() > 0);
+    }
+
+    @Test
+    public void matchInfoWithoutLogin () throws KustvaktException {
+        Response response = target().path(API_VERSION).path("corpus")
+                .path("WDD17").path("982").path("72848").path("p2815-2816")
+                .queryParam("foundry", "*").request().get();
+
+        assertEquals(Status.UNAUTHORIZED.getStatusCode(), response.getStatus());
+
+        String entity = response.readEntity(String.class);
+        JsonNode node = JsonUtils.readTree(entity);
+
+        assertEquals(StatusCodes.AUTHORIZATION_FAILED,
+                node.at("/errors/0/0").asInt());
+    }
+
+    @Test
+    public void matchInfoWithLogin () throws KustvaktException {
+        Response response = target().path(API_VERSION).path("corpus")
+                .path("WDD17").path("982").path("72848").path("p2815-2816")
+                .queryParam("foundry", "*").request()
+                .header(Attributes.AUTHORIZATION, basicAuth).get();
+
+        assertEquals(Status.OK.getStatusCode(), response.getStatus());
+
+        String entity = response.readEntity(String.class);
+        JsonNode node = JsonUtils.readTree(entity);
+
+        assertTrue(node.at("/hasSnippet").asBoolean());
+        assertNotNull(node.at("/matchID").asText());
+        assertNotNull(node.at("/snippet").asText());
+    }
+}
diff --git a/full/src/test/resources/kustvakt-icc.conf b/full/src/test/resources/kustvakt-icc.conf
new file mode 100644
index 0000000..c029395
--- /dev/null
+++ b/full/src/test/resources/kustvakt-icc.conf
@@ -0,0 +1,105 @@
+# index dir
+krill.indexDir = ../wiki-index
+
+krill.index.commit.count = 134217000
+krill.index.commit.log = log/krill.commit.log
+krill.index.commit.auto = 500
+krill.index.relations.max = 100
+# Directory path of virtual corpora to cache
+krill.namedVC = vc
+krill.test = true
+
+# LDAP
+ldap.config = src/test/resources/test-ldap.conf
+
+# Kustvakt
+# multiple versions separated by space
+current.api.version = v1.0
+supported.api.version = v0.1 v1.0
+
+# server
+server.port=8089
+server.host=localhost
+
+# mail settings
+mail.enabled = false
+mail.receiver = test@localhost
+mail.sender = noreply@ids-mannheim.de
+mail.address.retrieval = test
+
+# mail.templates
+template.group.invitation = notification.vm
+
+# default foundries for specific layers
+default.foundry.partOfSpeech = tt
+default.foundry.lemma = tt
+default.foundry.orthography = opennlp
+default.foundry.dependency = malt
+default.foundry.constituent = corenlp
+default.foundry.morphology = marmot
+default.foundry.surface = base
+
+# delete configuration (default hard)
+# delete.auto.group = hard
+delete.group = soft
+delete.group.member = soft
+
+
+# availability regex only support |
+# It should be removed/commented when the data doesn't contain availability field.
+# 
+# availability.regex.free = CC-BY.*
+# availability.regex.public = ACA.*|QAO-NC
+# availability.regex.all = QAO.*
+
+
+# Define resource filters for search and match info web-services
+#
+# AuthenticationFilter activates authentication using OAuth2 tokens
+# DemoUserFilter allows access to the services without login
+# 
+# Default values: AuthenticationFilter,DemoUserFilter
+#
+search.resource.filters=AuthenticationFilter
+
+
+# options referring to the security module!
+
+# OAuth
+# (see de.ids_mannheim.korap.constant.AuthenticationMethod for possible 
+# oauth.password.authentication values)
+oauth2.password.authentication = TEST
+oauth2.native.client.host = korap.ids-mannheim.de
+oauth2.max.attempts = 2
+# expiry in seconds (S), minutes (M), hours (H), days (D)
+oauth2.access.token.expiry = 3M
+oauth2.refresh.token.expiry = 90D
+oauth2.authorization.code.expiry = 10M
+# -- scopes separated by space
+oauth2.default.scopes = search match_info
+oauth2.client.credentials.scopes = client_info
+
+oauth2.initial.super.client=true
+
+# see SecureRandom Number Generation Algorithms
+# optional
+security.secure.random.algorithm=SHA1PRNG
+
+# see MessageDigest Algorithms
+# default MD5
+security.md.algoritm = SHA-256  
+
+# secure hash support: BCRYPT
+security.secure.hash.algorithm=BCRYPT
+security.encryption.loadFactor = 10
+
+
+# DEPRECATED
+# JWT 
+security.jwt.issuer=https://korap.ids-mannheim.de
+security.sharedSecret=this-is-shared-secret-code-for-JWT-Signing.It-must-contains-minimum-256-bits
+
+# token expiration time
+security.longTokenTTL = 1D
+security.tokenTTL = 2S
+security.shortTokenTTL = 1S
diff --git a/full/src/test/resources/kustvakt-test.conf b/full/src/test/resources/kustvakt-test.conf
index e72b1fd..430ad5f 100644
--- a/full/src/test/resources/kustvakt-test.conf
+++ b/full/src/test/resources/kustvakt-test.conf
@@ -1,36 +1,43 @@
-## index dir
+# Krill settings
+#
 krill.indexDir = ../sample-index
 
 krill.index.commit.count = 134217000
 krill.index.commit.log = log/krill.commit.log
 krill.index.commit.auto = 500
 krill.index.relations.max = 100
-## Directory path of virtual corpora to cache
+# Directory path of virtual corpora to cache
 krill.namedVC = vc
 krill.test = true
 
-#LDAP
+# LDAP configuration file
+#
 ldap.config = src/test/resources/test-ldap.conf
 
-# Kustvakt
+# Kustvakt versions
+#
 # multiple versions separated by space
 current.api.version = v1.0
 supported.api.version = v0.1 v1.0
 
-## server
+# Server
+#
 server.port=8089
 server.host=localhost
 
-## mail settings
+# Mail settings
+#
 mail.enabled = false
 mail.receiver = test@localhost
 mail.sender = noreply@ids-mannheim.de
 mail.address.retrieval = test
 
-## mail.templates
+# Mail.templates
+#
 template.group.invitation = notification.vm
 
-## default foundries for specific layers
+# Default foundries for specific layers (optional)
+#
 default.foundry.partOfSpeech = tt
 default.foundry.lemma = tt
 default.foundry.orthography = opennlp
@@ -39,22 +46,35 @@
 default.foundry.morphology = marmot
 default.foundry.surface = base
 
-## delete configuration (default hard)
+# Delete configuration (default hard)
+#
 # delete.auto.group = hard
 delete.group = soft
 delete.group.member = soft
 
-## availability regex
-## only support |
+# Availability regex only support |
+# It should be removed/commented when the data doesn't contain availability field.
+# 
 availability.regex.free = CC-BY.*
 availability.regex.public = ACA.*|QAO-NC
 availability.regex.all = QAO.*
 
-## options referring to the security module!
 
-## OAuth
-### (see de.ids_mannheim.korap.constant.AuthenticationMethod for possible 
-### oauth.password.authentication values)
+# Define resource filters for search and match info API
+# AuthenticationFilter activates authentication using OAuth2 tokens
+# DemoUserFilter allows access to API without login
+# 
+# Default values: AuthenticationFilter,DemoUserFilter
+#
+search.resource.filters=AuthenticationFilter,DemoUserFilter
+
+
+# options referring to the security module!
+
+# OAuth
+# (see de.ids_mannheim.korap.constant.AuthenticationMethod for possible 
+# oauth.password.authentication values)
+#
 oauth2.password.authentication = TEST
 oauth2.native.client.host = korap.ids-mannheim.de
 oauth2.max.attempts = 2
@@ -68,8 +88,8 @@
 
 oauth2.initial.super.client=true
 
-## OpenId
-### multiple values are separated by space
+# OpenId
+# multiple values are separated by space
 openid.grant.types = authorization_code
 openid.response.types = code
 openid.response.modes = query
@@ -86,29 +106,30 @@
 #openid.term.of.service =
 openid.service.doc = https://github.com/KorAP/Kustvakt/wiki
 
-## JWK
-## must be set for openid
+# JWK
+# must be set for openid
 rsa.private = kustvakt_rsa.key
 rsa.public = kustvakt_rsa_public.key
 rsa.key.id = 74caa3a9-217c-49e6-94e9-2368fdd02c35
 
-## see SecureRandom Number Generation Algorithms
-## optional
+# see SecureRandom Number Generation Algorithms
+# optional
 security.secure.random.algorithm=SHA1PRNG
 
-## see MessageDigest Algorithms
-## default MD5
+# see MessageDigest Algorithms
+# default MD5
 security.md.algoritm = SHA-256  
 
-### secure hash support: BCRYPT
+# secure hash support: BCRYPT
 security.secure.hash.algorithm=BCRYPT
 security.encryption.loadFactor = 10
 
-## JWT
+# DEPRECATED
+# JWT
 security.jwt.issuer=https://korap.ids-mannheim.de
 security.sharedSecret=this-is-shared-secret-code-for-JWT-Signing.It-must-contains-minimum-256-bits
 
-## token expiration time
+# token expiration time
 security.longTokenTTL = 1D
 security.tokenTTL = 2S
 security.shortTokenTTL = 1S
@@ -118,7 +139,3 @@
 security.multipleLogIn = true
 security.loginAttemptNum = 3
 security.authAttemptTTL = 45M
-
-#EM: deprecated and not used
-security.validation.stringLength = 150
-security.validation.emailLength = 50
diff --git a/full/src/test/resources/test-config-icc.xml b/full/src/test/resources/test-config-icc.xml
new file mode 100644
index 0000000..a98d1af
--- /dev/null
+++ b/full/src/test/resources/test-config-icc.xml
@@ -0,0 +1,358 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<beans xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+	xmlns:p="http://www.springframework.org/schema/p" xmlns:util="http://www.springframework.org/schema/util"
+	xmlns:aop="http://www.springframework.org/schema/aop" xmlns:tx="http://www.springframework.org/schema/tx"
+	xmlns="http://www.springframework.org/schema/beans" xmlns:context="http://www.springframework.org/schema/context"
+	xmlns:cache="http://www.springframework.org/schema/cache"
+	xsi:schemaLocation="http://www.springframework.org/schema/beans
+           http://www.springframework.org/schema/beans/spring-beans.xsd
+           http://www.springframework.org/schema/tx
+           http://www.springframework.org/schema/tx/spring-tx.xsd
+           http://www.springframework.org/schema/aop
+           http://www.springframework.org/schema/aop/spring-aop.xsd
+           http://www.springframework.org/schema/context
+           http://www.springframework.org/schema/context/spring-context.xsd
+           http://www.springframework.org/schema/util
+           http://www.springframework.org/schema/util/spring-util.xsd">
+
+	<context:component-scan base-package="de.ids_mannheim.korap" />
+	<context:annotation-config />
+
+	<bean id="props"
+		class="org.springframework.beans.factory.config.PropertiesFactoryBean">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="locations">
+			<array>
+				<value>file:./kustvakt-icc.conf</value>
+				<value>classpath:kustvakt-icc.conf</value>
+			</array>
+		</property>
+	</bean>
+
+	<bean id="placeholders"
+		class="org.springframework.context.support.PropertySourcesPlaceholderConfigurer">
+		<property name="ignoreResourceNotFound" value="true" />
+		<property name="locations">
+			<array>
+				<value>classpath:test-jdbc.properties</value>
+				<value>file:./jdbc.properties</value>
+				<value>classpath:properties/mail.properties</value>
+				<value>file:./mail.properties</value>
+				<value>classpath:hibernate.properties</value>
+				<value>file:./kustvakt-icc.conf</value>
+				<value>classpath:kustvakt-icc.conf</value>
+			</array>
+		</property>
+	</bean>
+
+	<bean id='cacheManager' class='org.springframework.cache.ehcache.EhCacheCacheManager'
+		p:cacheManager-ref='ehcache' />
+
+	<bean id='ehcache'
+		class='org.springframework.cache.ehcache.EhCacheManagerFactoryBean'
+		p:configLocation='classpath:ehcache.xml' p:shared='true' />
+
+	<bean id="dataSource" class="org.apache.commons.dbcp2.BasicDataSource"
+		lazy-init="true">
+		<!-- <property name="driverClassName" value="${jdbc.driverClassName}" /> -->
+		<property name="url" value="${jdbc.url}" />
+		<property name="username" value="${jdbc.username}" />
+		<property name="password" value="${jdbc.password}" />
+		<property name="maxTotal" value="4" />
+		<property name="maxIdle" value="1" />
+		<property name="minIdle" value="1" />
+		<property name="maxWaitMillis" value="15000" />
+		<!--<property name="poolPreparedStatements" value="true"/> -->
+	</bean>
+
+	<!-- use SingleConnection only for testing! -->
+	<bean id="sqliteDataSource"
+		class="org.springframework.jdbc.datasource.SingleConnectionDataSource"
+		lazy-init="true">
+		<!-- <property name="driverClassName" value="${jdbc.driverClassName}" /> -->
+		<property name="url" value="${jdbc.url}" />
+		<property name="username" value="${jdbc.username}" />
+		<property name="password" value="${jdbc.password}" />
+		<property name="connectionProperties">
+			<props>
+				<prop key="date_string_format">yyyy-MM-dd HH:mm:ss</prop>
+			</props>
+		</property>
+
+		<!-- Sqlite can only have a single connection -->
+		<property name="suppressClose">
+			<value>true</value>
+		</property>
+	</bean>
+
+	<bean id="c3p0DataSource" class="com.mchange.v2.c3p0.ComboPooledDataSource"
+		destroy-method="close">
+		<property name="driverClass" value="${jdbc.driverClassName}" />
+		<property name="jdbcUrl" value="${jdbc.url}" />
+		<property name="user" value="${jdbc.username}" />
+		<property name="password" value="${jdbc.password}" />
+		<property name="maxPoolSize" value="4" />
+		<property name="minPoolSize" value="1" />
+		<property name="maxStatements" value="1" />
+		<property name="testConnectionOnCheckout" value="true" />
+	</bean>
+
+	<!-- to configure database for sqlite, mysql, etc. migrations -->
+	<bean id="flywayConfig" class="org.flywaydb.core.api.configuration.ClassicConfiguration">
+		<!-- drop existing tables and create new tables -->
+		<property name="validateOnMigrate" value="true" />
+		<property name="cleanOnValidationError" value="true" />
+		<property name="baselineOnMigrate" value="false" />
+		<property name="locations" value="#{'${jdbc.schemaPath}'.split(',')}"/>
+		<property name="dataSource" ref="sqliteDataSource" />
+		<!-- <property name="dataSource" ref="dataSource" /> -->
+		<property name="outOfOrder" value="true" />
+	</bean>
+	
+	<bean id="flyway" class="org.flywaydb.core.Flyway" init-method="migrate">
+	    <constructor-arg ref="flywayConfig"/>
+	</bean>
+	
+
+	<bean id="kustvakt_db" class="de.ids_mannheim.korap.handlers.JDBCClient">
+		<!-- <constructor-arg index="0" ref="dataSource" /> -->
+		<constructor-arg index="0" ref="sqliteDataSource" />
+		<property name="database" value="${jdbc.database}" />
+	</bean>
+
+	<bean id="entityManagerFactory"
+		class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
+		<!-- <property name="dataSource" ref="dataSource" /> -->
+		<property name="dataSource" ref="sqliteDataSource" />
+		<property name="packagesToScan">
+			<array>
+				<value>de.ids_mannheim.korap.entity</value>
+				<value>de.ids_mannheim.korap.oauth2.entity</value>
+			</array>
+		</property>
+		<property name="jpaVendorAdapter">
+			<bean id="jpaVendorAdapter"
+				class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
+				<property name="databasePlatform" value="${hibernate.dialect}" />
+			</bean>
+		</property>
+		<property name="jpaProperties">
+			<props>
+				<prop key="hibernate.dialect">${hibernate.dialect}</prop>
+				<prop key="hibernate.hbm2ddl.auto">${hibernate.hbm2ddl.auto}</prop>
+				<prop key="hibernate.show_sql">${hibernate.show_sql}</prop>
+				<prop key="hibernate.cache.use_query_cache">${hibernate.cache.use_query_cache}</prop>
+				<prop key="hibernate.cache.use_second_level_cache">${hibernate.cache.use_second_level_cache}
+				</prop>
+				<prop key="hibernate.cache.provider_class">${hibernate.cache.provider}</prop>
+				<prop key="hibernate.cache.region.factory_class">${hibernate.cache.region.factory}</prop>
+				<prop key="hibernate.jdbc.time_zone">${hibernate.jdbc.time_zone}</prop>
+				<!-- <prop key="net.sf.ehcache.configurationResourceName">classpath:ehcache.xml</prop> -->
+			</props>
+		</property>
+	</bean>
+
+	<tx:annotation-driven proxy-target-class="true"
+		transaction-manager="transactionManager" />
+	<bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
+		<property name="entityManagerFactory" ref="entityManagerFactory" />
+	</bean>
+
+	<bean id="transactionTemplate"
+		class="org.springframework.transaction.support.TransactionTemplate">
+		<constructor-arg ref="transactionManager" />
+	</bean>
+
+	<!-- Data access objects -->
+	<bean id="adminDao" class="de.ids_mannheim.korap.dao.AdminDaoImpl" />
+	<bean id="resourceDao" class="de.ids_mannheim.korap.dao.ResourceDao" />
+	<bean id="accessScopeDao" class="de.ids_mannheim.korap.oauth2.dao.AccessScopeDao" />
+	<bean id="authorizationDao" class="de.ids_mannheim.korap.oauth2.dao.CachedAuthorizationDaoImpl" />
+	
+	<!-- Services -->
+	<bean id="scopeService" class="de.ids_mannheim.korap.oauth2.service.OAuth2ScopeServiceImpl" />
+	
+	
+	<!-- Controller -->
+	
+	
+	<!-- props are injected from default-config.xml -->
+	<bean id="kustvakt_config" class="de.ids_mannheim.korap.config.FullConfiguration">
+		<constructor-arg name="properties" ref="props" />
+	</bean>
+
+	<bean id="initializator" class="de.ids_mannheim.de.init.Initializator"
+		init-method="initTest">
+	</bean>
+
+	<!-- Krill -->
+	<bean id="search_krill" class="de.ids_mannheim.korap.web.SearchKrill">
+		<constructor-arg value="${krill.indexDir}" />
+	</bean>
+
+	<!-- Validator -->
+	<bean id="validator" class="de.ids_mannheim.korap.validator.ApacheValidator"/>
+	
+	<!-- URLValidator -->
+	<bean id="redirectURIValidator" class="org.apache.commons.validator.routines.UrlValidator">
+		<constructor-arg value="http,https" index="0" />
+		<constructor-arg index="1" type="long" 
+		value="#{T(org.apache.commons.validator.routines.UrlValidator).ALLOW_LOCAL_URLS + 
+		T(org.apache.commons.validator.routines.UrlValidator).NO_FRAGMENTS}"/>
+	</bean>
+	<bean id="urlValidator" class="org.apache.commons.validator.routines.UrlValidator">
+		<constructor-arg value="http,https" />
+	</bean>
+
+	<!-- Rewrite -->
+	<bean id="foundryRewrite" class="de.ids_mannheim.korap.rewrite.FoundryRewrite"/>
+	<bean id="collectionRewrite" class="de.ids_mannheim.korap.rewrite.CollectionRewrite"/>
+	<bean id="collectionCleanRewrite" class="de.ids_mannheim.korap.rewrite.CollectionCleanRewrite"/>
+	<bean id="virtualCorpusRewrite" class="de.ids_mannheim.korap.rewrite.VirtualCorpusRewrite"/>
+	<bean id="collectionConstraint" class="de.ids_mannheim.korap.rewrite.CollectionConstraint"/>
+ 	<bean id="queryReferenceRewrite" class="de.ids_mannheim.korap.rewrite.QueryReferenceRewrite"/>
+	
+	<util:list id="rewriteTasks"
+		value-type="de.ids_mannheim.korap.rewrite.RewriteTask">
+		<!-- <ref bean="collectionConstraint" />
+		<ref bean="collectionCleanRewrite" /> -->
+		<ref bean="foundryRewrite" />
+		<!-- <ref bean="collectionRewrite" /> -->
+		<ref bean="virtualCorpusRewrite" />
+		<ref bean="queryReferenceRewrite" />
+	</util:list>
+	
+	<bean id="rewriteHandler" class="de.ids_mannheim.korap.rewrite.RewriteHandler">
+		<constructor-arg ref="rewriteTasks"/>
+	</bean>
+
+	<bean id="kustvakt_auditing" class="de.ids_mannheim.korap.handlers.JDBCAuditing">
+		<constructor-arg ref="kustvakt_db" />
+	</bean>
+
+	<bean id="kustvaktResponseHandler" class="de.ids_mannheim.korap.web.KustvaktResponseHandler">
+		<constructor-arg index="0" name="iface" ref="kustvakt_auditing" />
+	</bean>
+
+	<!-- OAuth -->
+	<bean id="oauth2ResponseHandler" class="de.ids_mannheim.korap.web.OAuth2ResponseHandler">
+		<constructor-arg index="0" name="iface" ref="kustvakt_auditing" />
+	</bean>
+
+	<bean id="mdGenerator" class="org.apache.oltu.oauth2.as.issuer.MD5Generator">
+	</bean>
+	<bean id="oauthIssuer" class="org.apache.oltu.oauth2.as.issuer.OAuthIssuerImpl">
+		<constructor-arg index="0" ref="mdGenerator" />
+	</bean>
+
+	<bean id="kustvakt_userdb" class="de.ids_mannheim.korap.handlers.EntityDao">
+		<constructor-arg ref="kustvakt_db" />
+	</bean>
+
+	<bean name="kustvakt_encryption" class="de.ids_mannheim.korap.encryption.KustvaktEncryption">
+		<constructor-arg ref="kustvakt_config" />
+	</bean>
+
+	<!-- authentication providers to use -->
+	<bean id="basic_auth"
+		class="de.ids_mannheim.korap.authentication.BasicAuthentication" />
+
+	<bean id="oauth2_auth"
+		class="de.ids_mannheim.korap.authentication.OAuth2Authentication" />
+
+
+	<util:list id="kustvakt_authproviders"
+		value-type="de.ids_mannheim.korap.interfaces.AuthenticationIface">
+		<ref bean="basic_auth" />
+		<ref bean="oauth2_auth" />
+	</util:list>
+
+
+	<bean id="userdata_details" class="de.ids_mannheim.korap.handlers.UserDetailsDao">
+		<constructor-arg ref="kustvakt_db" />
+	</bean>
+
+	<bean id="userdata_settings" class="de.ids_mannheim.korap.handlers.UserSettingsDao">
+		<constructor-arg ref="kustvakt_db" />
+	</bean>
+
+	<util:list id="kustvakt_userdata"
+		value-type="de.ids_mannheim.korap.interfaces.db.UserdataDbIface">
+		<ref bean="userdata_details" />
+		<ref bean="userdata_settings" />
+	</util:list>
+
+	<!-- specify type for constructor argument -->
+	<bean id="authenticationManager"
+		class="de.ids_mannheim.korap.authentication.KustvaktAuthenticationManager">
+		<constructor-arg type="de.ids_mannheim.korap.interfaces.EntityHandlerIface"
+			ref="kustvakt_userdb" />
+		<constructor-arg type="de.ids_mannheim.korap.interfaces.EncryptionIface"
+			ref="kustvakt_encryption" />
+		<constructor-arg ref="kustvakt_config" />
+		<constructor-arg type="de.ids_mannheim.korap.interfaces.db.AuditingIface"
+			ref="kustvakt_auditing" />
+		<constructor-arg ref="kustvakt_userdata" />
+		<!-- inject authentication providers to use -->
+		<property name="providers" ref="kustvakt_authproviders" />
+	</bean>
+
+	<!-- todo: if db interfaces not loaded via spring, does transaction even 
+		work then? -->
+	<!-- the transactional advice (i.e. what 'happens'; see the <aop:advisor/> 
+		bean below) -->
+	<tx:advice id="txAdvice" transaction-manager="txManager">
+		<!-- the transactional semantics... -->
+		<tx:attributes>
+			<!-- all methods starting with 'get' are read-only -->
+			<tx:method name="get*" read-only="true" rollback-for="KorAPException" />
+			<!-- other methods use the default transaction settings (see below) -->
+			<tx:method name="*" rollback-for="KorAPException" />
+		</tx:attributes>
+	</tx:advice>
+
+	<!-- ensure that the above transactional advice runs for any execution of 
+		an operation defined by the service interface -->
+	<aop:config>
+		<aop:pointcut id="service"
+			expression="execution(* de.ids_mannheim.korap.interfaces.db.*.*(..))" />
+		<aop:advisor advice-ref="txAdvice" pointcut-ref="service" />
+	</aop:config>
+
+	<!-- similarly, don't forget the PlatformTransactionManager -->
+	<bean id="txManager"
+		class="org.springframework.jdbc.datasource.DataSourceTransactionManager">
+		<property name="dataSource" ref="dataSource" />
+	</bean>
+
+	<!-- mail -->
+	<bean id="authenticator" class="de.ids_mannheim.korap.service.MailAuthenticator">
+		<constructor-arg index="0" value="${mail.username}" />
+		<constructor-arg index="1" value="${mail.password}" />
+	</bean>
+	<bean id="smtpSession" class="javax.mail.Session" factory-method="getInstance">
+		<constructor-arg index="0">
+			<props>
+				<prop key="mail.smtp.submitter">${mail.username}</prop>
+				<prop key="mail.smtp.auth">${mail.auth}</prop>
+				<prop key="mail.smtp.host">${mail.host}</prop>
+				<prop key="mail.smtp.port">${mail.port}</prop>
+				<prop key="mail.smtp.starttls.enable">${mail.starttls.enable}</prop>
+				<prop key="mail.smtp.connectiontimeout">${mail.connectiontimeout}</prop>
+			</props>
+		</constructor-arg>
+		<constructor-arg index="1" ref="authenticator" />
+	</bean>
+	<bean id="mailSender" class="org.springframework.mail.javamail.JavaMailSenderImpl">
+		<property name="session" ref="smtpSession" />
+	</bean>
+	<bean id="velocityEngine" class="org.apache.velocity.app.VelocityEngine">
+		<constructor-arg index="0">
+			<props>
+				<prop key="resource.loader">class</prop>
+				<prop key="class.resource.loader.class">org.apache.velocity.runtime.resource.loader.ClasspathResourceLoader
+				</prop>
+			</props>
+		</constructor-arg>
+	</bean>
+</beans>
diff --git a/full/src/test/resources/test-config.xml b/full/src/test/resources/test-config.xml
index aaea39c..cffc34e 100644
--- a/full/src/test/resources/test-config.xml
+++ b/full/src/test/resources/test-config.xml
@@ -39,6 +39,7 @@
 				<value>classpath:properties/mail.properties</value>
 				<value>file:./mail.properties</value>
 				<value>classpath:test-hibernate.properties</value>
+				<value>file:./kustvakt-test.conf</value>
 				<value>classpath:kustvakt-test.conf</value>
 			</array>
 		</property>
diff --git a/wiki-index/_0.cfe b/wiki-index/_0.cfe
new file mode 100644
index 0000000..17325ac
--- /dev/null
+++ b/wiki-index/_0.cfe
Binary files differ
diff --git a/wiki-index/_0.cfs b/wiki-index/_0.cfs
new file mode 100644
index 0000000..65b7ade
--- /dev/null
+++ b/wiki-index/_0.cfs
Binary files differ
diff --git a/wiki-index/_0.si b/wiki-index/_0.si
new file mode 100644
index 0000000..1d5b267
--- /dev/null
+++ b/wiki-index/_0.si
Binary files differ
diff --git a/wiki-index/segments_1 b/wiki-index/segments_1
new file mode 100644
index 0000000..4df408b
--- /dev/null
+++ b/wiki-index/segments_1
Binary files differ
diff --git a/wiki-index/write.lock b/wiki-index/write.lock
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/wiki-index/write.lock