| Akron | 8b511f9 | 2020-07-09 17:28:08 +0200 | [diff] [blame] | 1 | #!/usr/bin/env perl |
| 2 | use strict; |
| 3 | use warnings; |
| 4 | use FindBin; |
| 5 | BEGIN { |
| 6 | unshift @INC, "$FindBin::Bin/../../lib"; |
| 7 | }; |
| 8 | use KorAP::XML::TEI::Tokenizer::Aggressive; |
| 9 | |
| 10 | use open qw(:std :utf8); # assume utf-8 encoding |
| 11 | |
| 12 | $| = 1; |
| 13 | |
| 14 | # Init tokenizer |
| 15 | my $tok = KorAP::XML::TEI::Tokenizer::Aggressive->new; |
| 16 | |
| 17 | # Read lines from input and return boundaries |
| 18 | while (!eof(STDIN)) { |
| 19 | my $line = <>; |
| 20 | $tok->tokenize($line); |
| 21 | print join(' ', $tok->boundaries), "\n"; |
| 22 | $tok->reset; |
| 23 | }; |
| 24 | |
| 25 | 1; |