Bugfix: intern tokenization
retr_info(): $_GEN_TOK_INT: text starting with 2 blanks is ignored
Change-Id: I06e8a157bc566fca4b44737230f9b3dc236d0a98
diff --git a/t/tokenization.t b/t/tokenization.t
index 5332196..dfe05bb 100644
--- a/t/tokenization.t
+++ b/t/tokenization.t
@@ -29,6 +29,9 @@
$cons->reset->tokenize("Der alte bzw. der grau-melierte Mann");
is_deeply($cons, [0,3,4,8,9,12,12,13,14,17,18,31,32,36]);
+$cons->reset->tokenize(" Der alte bzw. der grau-melierte Mann");
+is_deeply($cons, [2,5,6,10,11,14,14,15,16,19,20,33,34,38]);
+
$cons->reset->tokenize(". Der");
is_deeply($cons, [0,1,2,5]);