Fix token column in corpusQuery results

Change-Id: I439480381684abd0a9dbaa9bf19feff06c62fd2c
diff --git a/R/KorAPQuery.R b/R/KorAPQuery.R
index a9882cb..8e72b60 100644
--- a/R/KorAPQuery.R
+++ b/R/KorAPQuery.R
@@ -323,6 +323,9 @@
       if("snippet" %in% colnames(res$matches)) {
         currentMatches$snippet <- res$matches$snippet
       }
+      if ("tokens" %in% colnames(res$matches)) {
+        currentMatches$tokens <- res$matches$tokens
+      }
     } else {
       currentMatches <- res$matches
     }
diff --git a/tests/testthat/test-corpusQuery.R b/tests/testthat/test-corpusQuery.R
index b9d4b76..13e2d51 100644
--- a/tests/testthat/test-corpusQuery.R
+++ b/tests/testthat/test-corpusQuery.R
@@ -116,3 +116,22 @@
   kco <- new("KorAPConnection", verbose = TRUE)
   expect_message(kco %>% corpusQuery("[[xx"), "unbalanced")
 })
+
+test_that("corpusQuery token API works", {
+  skip_if_offline()
+  kco <- new("KorAPConnection", accessToken = NULL,
+             verbose = TRUE)
+  q <- corpusQuery(kco, "focus([tt/p=ADJA] {Newstickeritis})", vc = "corpusSigle=/W.D17/", metadataOnly = FALSE)
+  q <- fetchNext(q)
+  matches <-q@collectedMatches
+  expect_gt(nrow(matches), 10)
+  unique_matches <- unique(matches$tokens$match)
+  expect_equal(length(unique_matches), 1)
+  expect_equal(unique_matches[[1]], "Newstickeritis")
+
+  left_contexts <- matches$tokens$left
+  expect_true(TRUE %in% grepl("reine", left_contexts))
+
+  right_contexts <- matches$tokens$right
+  expect_true(TRUE %in% grepl("Begriff", right_contexts))
+})