| Akron | 8b511f9 | 2020-07-09 17:28:08 +0200 | [diff] [blame] | 1 | #!/usr/bin/env perl |
| 2 | use strict; |
| 3 | use warnings; |
| 4 | use FindBin; |
| Marc Kupietz | 1e882fb | 2020-09-09 00:05:46 +0200 | [diff] [blame^] | 5 | use Encode; |
| Akron | 8b511f9 | 2020-07-09 17:28:08 +0200 | [diff] [blame] | 6 | BEGIN { |
| 7 | unshift @INC, "$FindBin::Bin/../../lib"; |
| 8 | }; |
| 9 | use KorAP::XML::TEI::Tokenizer::Aggressive; |
| 10 | |
| Akron | 8b511f9 | 2020-07-09 17:28:08 +0200 | [diff] [blame] | 11 | $| = 1; |
| 12 | |
| 13 | # Init tokenizer |
| 14 | my $tok = KorAP::XML::TEI::Tokenizer::Aggressive->new; |
| 15 | |
| 16 | # Read lines from input and return boundaries |
| 17 | while (!eof(STDIN)) { |
| Marc Kupietz | 1e882fb | 2020-09-09 00:05:46 +0200 | [diff] [blame^] | 18 | my $line = decode_utf8(<>); |
| Marc Kupietz | 52dc21b | 2020-09-05 13:51:22 +0200 | [diff] [blame] | 19 | for my $text (split(/\n?\x{04}\n?/, $line)) { |
| 20 | $tok->tokenize($text); |
| 21 | print join(' ', $tok->boundaries), "\n"; |
| 22 | $tok->reset; |
| 23 | } |
| Akron | 8b511f9 | 2020-07-09 17:28:08 +0200 | [diff] [blame] | 24 | }; |
| 25 | |
| 26 | 1; |