Make tokenization chainable and remove unnecessary tokenization switch
Change-Id: Iaf47a0fcad225931c941fd7379c02e51a428be07
diff --git a/t/tokenization.t b/t/tokenization.t
index b132a63..92b7cc3 100644
--- a/t/tokenization.t
+++ b/t/tokenization.t
@@ -24,6 +24,12 @@
$aggr->reset->tokenize("Der alte bzw. der grau-melierte Mann");
is_deeply($aggr, [0,3,4,8,9,12,12,13,14,17,18,22,22,23,23,31,32,36]);
+like(
+ $aggr->reset->tokenize("Der")->to_string('a'),
+ qr!id="t_0"!,
+ 'Chainable'
+);
+
# Test conservative
my $cons = KorAP::XML::TEI::Tokenizer::Conservative->new;
$cons->tokenize("Der alte Mann");