Support tokens in match information (fixes #570)

Change-Id: If5d4289eebad0ee6ed71079ecebe01b2412be554
diff --git a/lite/Changes b/lite/Changes
index 7808c66..9168262 100644
--- a/lite/Changes
+++ b/lite/Changes
@@ -1,5 +1,7 @@
 # version 0.69.4
 
+- Support token array in matchinfo (fixes #570; diewald)
+
 # version 0.69.3
 
 # version 0.69.2
diff --git a/lite/src/test/java/de/ids_mannheim/korap/web/service/LiteSearchControllerTest.java b/lite/src/test/java/de/ids_mannheim/korap/web/service/LiteSearchControllerTest.java
index 9378ed8..f120ff2 100644
--- a/lite/src/test/java/de/ids_mannheim/korap/web/service/LiteSearchControllerTest.java
+++ b/lite/src/test/java/de/ids_mannheim/korap/web/service/LiteSearchControllerTest.java
@@ -347,6 +347,48 @@
     }
 
     @Test
+    public void testTokenRetrieval () throws KustvaktException {
+        Response response =
+                target().path(API_VERSION).path("/corpus/GOE/AGA/01784/p104-105/")
+                        .request()
+                        .method("GET");
+        assertEquals(Status.OK.getStatusCode(),
+                response.getStatus());
+        String resp = response.readEntity(String.class);
+        JsonNode node = JsonUtils.readTree(resp);
+        assertTrue(node.at("/hasSnippet").asBoolean());
+        assertFalse(node.at("/hasTokens").asBoolean());
+        assertTrue(node.at("/tokens").isMissingNode());
+        assertEquals(
+            "<span class=\"context-left\"><span class=\"more\"></span></span>"+
+            "<span class=\"match\"><mark>die</mark></span>"+
+            "<span class=\"context-right\"><span class=\"more\"></span></span>",
+            node.at("/snippet").asText());
+        
+        // Tokens
+        response =
+             target().path(API_VERSION).path("/corpus/GOE/AGA/01784/p104-105")
+             .queryParam("show-snippet", "false")
+             .queryParam("show-tokens", "true")
+             .queryParam("expand", "false")
+             .request()
+             .method("GET");
+        assertEquals(Status.OK.getStatusCode(),
+                response.getStatus());
+        resp = response.readEntity(String.class);
+        node = JsonUtils.readTree(resp);
+
+        assertFalse(node.at("/hasSnippet").asBoolean());
+        assertTrue(node.at("/hasTokens").asBoolean());
+        assertTrue(node.at("/snippet").isMissingNode());
+        assertEquals(
+            "die",
+            node.at("/tokens/match/0").asText());
+        assertTrue(node.at("/tokens/match/1").isMissingNode());
+    };
+
+    
+    @Test
     public void testMetaFields () throws KustvaktException {
         Response response =
                 target().path(API_VERSION).path("/corpus/GOE/AGA/01784")