Akron | 414ec95 | 2020-08-03 15:48:43 +0200 | [diff] [blame^] | 1 | use strict; |
| 2 | use warnings; |
| 3 | use Test::More; |
| 4 | use File::Basename 'dirname'; |
| 5 | use File::Spec::Functions 'catdir'; |
| 6 | use Data::Dumper; |
| 7 | use KorAP::XML::Tokenizer; |
| 8 | use KorAP::XML::Krill; |
| 9 | use utf8; |
| 10 | |
| 11 | if ($ENV{SKIP_REAL}) { |
| 12 | plan skip_all => 'Skip real tests'; |
| 13 | }; |
| 14 | |
| 15 | my $path = catdir(dirname(__FILE__), 'TEST', 'BSP', 1); |
| 16 | |
| 17 | ok(my $doc = KorAP::XML::Krill->new( |
| 18 | path => $path . '/' |
| 19 | ), 'Create Document'); |
| 20 | |
| 21 | ok($doc->parse, 'Parse document'); |
| 22 | |
| 23 | ok(my $tokens = KorAP::XML::Tokenizer->new( |
| 24 | path => $doc->path, |
| 25 | doc => $doc, |
| 26 | foundry => 'Sgbr', |
| 27 | layer => 'Lemma', |
| 28 | name => 'tokens' |
| 29 | ), 'Create tokens based on lemmata'); |
| 30 | |
| 31 | ok($tokens->parse, 'Parse tokenization based on lemmata'); |
| 32 | |
| 33 | my $data = $tokens->to_data->{data}; |
| 34 | |
| 35 | my $stream = $data->{stream}; |
| 36 | |
| 37 | is($stream->[0]->[0], '-:tokens$<i>51', 'Token number'); |
| 38 | is($stream->[0]->[2], '_0$<i>0<i>18', 'Position'); |
| 39 | is($stream->[0]->[3], 'i:sommerüberraschung', 'First term'); |
| 40 | is($stream->[0]->[4], 's:Sommerüberraschung', 'First term'); |
| 41 | is($stream->[-1]->[0], '_50$<i>359<i>364', 'Last position'); |
| 42 | is($stream->[-1]->[1], 'i:kevin', 'Last term'); |
| 43 | is($stream->[-1]->[2], 's:Kevin', 'Last term'); |
| 44 | |
| 45 | done_testing; |