Establish tokenizer object for external base tokenization
Change-Id: Ie69c280042da5125e0934c87ccaad88b0be5494f
diff --git a/t/cmd/tokenizer.pl b/t/cmd/tokenizer.pl
new file mode 100644
index 0000000..e484160
--- /dev/null
+++ b/t/cmd/tokenizer.pl
@@ -0,0 +1,25 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+use FindBin;
+BEGIN {
+ unshift @INC, "$FindBin::Bin/../../lib";
+};
+use KorAP::XML::TEI::Tokenizer::Aggressive;
+
+use open qw(:std :utf8); # assume utf-8 encoding
+
+$| = 1;
+
+# Init tokenizer
+my $tok = KorAP::XML::TEI::Tokenizer::Aggressive->new;
+
+# Read lines from input and return boundaries
+while (!eof(STDIN)) {
+ my $line = <>;
+ $tok->tokenize($line);
+ print join(' ', $tok->boundaries), "\n";
+ $tok->reset;
+};
+
+1;