blob: f9c336b0e6c8fccfe9fc88e2ab80870a50d88dcd [file] [log] [blame]
Akron797e8072020-02-13 07:59:40 +01001use strict;
2use warnings;
3use File::Basename 'dirname';
4use File::Spec::Functions qw/catfile/;
Peter Harders1c5ce152020-07-22 18:02:50 +02005use Encode qw!encode_utf8 decode_utf8 encode!;
Akron797e8072020-02-13 07:59:40 +01006
7use Test::More;
8use Test::Output;
Peter Harders42e18a62020-07-21 02:43:26 +02009
10use FindBin;
11BEGIN {
12 unshift @INC, "$FindBin::Bin/../lib";
13};
Peter Harders1c5ce152020-07-22 18:02:50 +020014
Akron044bd922020-07-30 09:12:28 +020015use Test::KorAP::XML::TEI qw!korap_tempfile i5_template test_tei2korapxml!;
Peter Harders57c884e2020-07-16 01:28:52 +020016
Akron797e8072020-02-13 07:59:40 +010017my $f = dirname(__FILE__);
18my $script = catfile($f, '..', 'script', 'tei2korapxml');
19ok(-f $script, 'Script found');
20
Akrond949e182020-02-14 12:23:57 +010021stdout_like(
Akron797e8072020-02-13 07:59:40 +010022 sub { system('perl', $script, '--help') },
Akrond949e182020-02-14 12:23:57 +010023 qr!This\s*program\s*is\s*usually\s*called\s*from\s*inside\s*another\s*script\.!,
Akron797e8072020-02-13 07:59:40 +010024 'Help'
25);
26
Akron044bd922020-07-30 09:12:28 +020027
Akrond949e182020-02-14 12:23:57 +010028stdout_like(
29 sub { system('perl', $script, '--version') },
30 qr!tei2korapxml - v\d+?\.\d+?!,
31 'Version'
32);
33
Akron2a60c532020-02-13 15:52:18 +010034# Load example file
35my $file = catfile($f, 'data', 'goe_sample.i5.xml');
Peter Harders57c884e2020-07-16 01:28:52 +020036
Akronb3649472020-09-29 08:24:46 +020037subtest 'Debugging' => sub {
38
39 my $t = test_tei2korapxml(
40 tmp => 'script_out',
41 file => $file,
42 param => '-ti',
43 env => 'KORAPXMLTEI_DEBUG=1'
44 )->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
45 ->stderr_like(qr!Debugging is activated!);
46};
47
Akron044bd922020-07-30 09:12:28 +020048subtest 'Basic processing' => sub {
Akron2a60c532020-02-13 15:52:18 +010049
Akron044bd922020-07-30 09:12:28 +020050 my $t = test_tei2korapxml(
51 tmp => 'script_out',
52 file => $file,
53 param => '-ti'
Akronb3649472020-09-29 08:24:46 +020054 )->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
55 ->stderr_unlike(qr!Debugging is activated!);
Akron2a60c532020-02-13 15:52:18 +010056
Akron68966082020-02-13 15:52:18 +010057
Akron044bd922020-07-30 09:12:28 +020058 # Uncompress GOE/header.xml from zip file
59 $t->unzip_xml('GOE/header.xml')
Akron2a60c532020-02-13 15:52:18 +010060
Akron044bd922020-07-30 09:12:28 +020061 # TODO: check wrong encoding in header-files (compare with input document)!
62 ->text_is('korpusSigle', 'GOE', 'korpusSigle')
63 ->text_is('h\.title[type=main]', 'Goethes Werke', 'h.title')
64 ->text_is('h\.author', 'Goethe, Johann Wolfgang von', 'h.author')
65 ->text_is('pubDate[type=year]', '1982', 'pubDate');
Akron2a60c532020-02-13 15:52:18 +010066
Akron044bd922020-07-30 09:12:28 +020067 # Uncompress GOE/AGA/header.xml from zip file
68 $t->unzip_xml('GOE/AGA/header.xml')
69 ->text_is('dokumentSigle', 'GOE/AGA', 'dokumentSigle')
70 ->text_is('d\.title', 'Goethe: Autobiographische Schriften II, (1817-1825, 1832)', 'd.title')
71 ->text_is('creatDate', '1820-1822', 'creatDate');
Akron2a60c532020-02-13 15:52:18 +010072
Akron044bd922020-07-30 09:12:28 +020073 # Uncompress GOE/AGA/00000/header.xml from zip file
74 $t->unzip_xml('GOE/AGA/00000/header.xml')
75 ->text_is('textSigle', 'GOE/AGA.00000', 'textSigle')
76 ->text_is('analytic > h\.title[type=main]', 'Campagne in Frankreich', 'h.title');
Akron2a60c532020-02-13 15:52:18 +010077
78# Uncompress GOE/AGA/00000/data.xml from zip file
Akron044bd922020-07-30 09:12:28 +020079 $t->unzip_xml('GOE/AGA/00000/data.xml')
80 ->attr_is('raw_text', 'docid', 'GOE_AGA.00000', 'text id')
Marc Kupietzfd0e6a92020-09-09 18:07:29 +020081 ->text_like('raw_text > text', qr!^Campagne in Frankreich 1792.*?uns allein begl.*cke\.$!, 'text content')
82 ->text_like('raw_text > text', qr!unter dem Titel "Kriegstheater"!, 'text content');
Akron2a60c532020-02-13 15:52:18 +010083
Akron417ed2f2020-09-28 12:43:17 +020084 my $content = $t->get_content_of('GOE/AGA/00000/data.xml');
85 like($content, qr!unter dem Titel "Kriegstheater"!, 'raw text content');
86
Akron044bd922020-07-30 09:12:28 +020087 $t->unzip_xml('GOE/AGA/00000/struct/structure.xml')
Akronc57625b2020-07-30 11:46:06 +020088 ->text_is('span[id=s3] *[name=type]', 'Autobiographie', 'text content')
89 ->text_is('#s3 *[name=type]', 'Autobiographie', 'text content')
90 ->attr_is('#s0','to','1266')
91 ->attr_is('#s0','l','1')
92
93 ->attr_is('#s18','from','925')
94 ->attr_is('#s18','to','1266')
95 ->attr_is('#s18','l','5')
96 ->attr_is('#s18 > fs','type', 'struct')
97 ->attr_is('#s18 > fs > f','name','name')
98 ->text_is('#s18 > fs > f','poem')
99
100 ->attr_is('#s19','from','925')
101 ->attr_is('#s19','to','1098')
102 ->attr_is('#s19','l','6')
103 ->attr_is('#s19 > fs','type','struct')
104 ->text_is('#s19 > fs > f[name=name]','lg')
105 ->text_is('#s19 > fs > f[name=attr] > fs[type=attr] >f[name=part]','u')
106
107 ->attr_is('#s37','from','1229')
108 ->attr_is('#s37','to','1266')
109 ->attr_is('#s37','l','8')
110 ->attr_is('#s37 > fs','type','struct')
111 ->text_is('#s37 > fs > f[name=name]','s')
112 ->text_is('#s37 > fs > f[name=attr] > fs[type=attr] > f[name=type]','manual')
113
114 ->attr_is('#s38','from','1266')
115 ->attr_is('#s38','to','1266')
116 ->attr_is('#s38','l','2')
117 ->attr_is('#s38 > fs','type','struct')
118 ->text_is('#s38 > fs > f[name=name]','back')
Akronf7084c42021-01-07 10:25:22 +0100119 ->element_count_is('*', 196);
Akron2a60c532020-02-13 15:52:18 +0100120
Akron044bd922020-07-30 09:12:28 +0200121 $t->file_exists_not('GOE/AGA/00000/base/tokens.xml', 'External not generated');
Akron2a60c532020-02-13 15:52:18 +0100122
Akron044bd922020-07-30 09:12:28 +0200123 # Uncompress GOE/AGA/00000/base/tokens_aggressive.xml from zip file
124 $t->unzip_xml('GOE/AGA/00000/base/tokens_aggressive.xml')
125 ->attr_is('spanList span:nth-child(1)', 'to', 8)
126 ->attr_is('spanList span#t_1', 'from', 9)
127 ->attr_is('spanList span#t_1', 'to', 11)
Akron2a60c532020-02-13 15:52:18 +0100128
Akron044bd922020-07-30 09:12:28 +0200129 ->attr_is('spanList span#t_67', 'from', 427)
130 ->attr_is('spanList span#t_67', 'to', 430)
Akron2a60c532020-02-13 15:52:18 +0100131
Akron044bd922020-07-30 09:12:28 +0200132 ->attr_is('spanList span#t_214', 'from', 1209)
133 ->attr_is('spanList span#t_214', 'to', 1212)
Akron2a60c532020-02-13 15:52:18 +0100134
Akron044bd922020-07-30 09:12:28 +0200135 ->element_count_is('spanList span', 227);
Peter Harders57c884e2020-07-16 01:28:52 +0200136
Akron044bd922020-07-30 09:12:28 +0200137 # Uncompress GOE/AGA/00000/base/tokens_conservative.xml from zip file
138 $t->unzip_xml('GOE/AGA/00000/base/tokens_conservative.xml')
139 ->attr_is('spanList span:nth-child(1)', 'to', 8)
Akron2a60c532020-02-13 15:52:18 +0100140
Akron044bd922020-07-30 09:12:28 +0200141 ->attr_is('spanList span#t_1', 'from', 9)
142 ->attr_is('spanList span#t_1', 'to', 11)
Akron797e8072020-02-13 07:59:40 +0100143
Akron044bd922020-07-30 09:12:28 +0200144 ->attr_is('spanList span#t_67', 'from', 427)
145 ->attr_is('spanList span#t_67', 'to', 430)
Akroneac374d2020-07-07 09:00:44 +0200146
Akron044bd922020-07-30 09:12:28 +0200147 ->attr_is('spanList span#t_214', 'from', 1209)
148 ->attr_is('spanList span#t_214', 'to', 1212)
Akroneac374d2020-07-07 09:00:44 +0200149
Akron044bd922020-07-30 09:12:28 +0200150 ->element_count_is('spanList span', 227);
151};
Akroneac374d2020-07-07 09:00:44 +0200152
Akroneac374d2020-07-07 09:00:44 +0200153
Akron044bd922020-07-30 09:12:28 +0200154subtest 'Tokenize with external tokenizer' => sub {
Akroneac374d2020-07-07 09:00:44 +0200155
Akron044bd922020-07-30 09:12:28 +0200156 my $cmd = catfile($f, 'cmd', 'tokenizer.pl');
Akroneac374d2020-07-07 09:00:44 +0200157
Akron044bd922020-07-30 09:12:28 +0200158 test_tei2korapxml(
159 file => $file,
160 param => "-tc='perl $cmd'",
161 tmp => 'script_out2'
162 )
163 ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
164 ->file_readable('GOE/AGA/00000/base/tokens.xml')
Akroneac374d2020-07-07 09:00:44 +0200165
Akron044bd922020-07-30 09:12:28 +0200166 # Uncompress GOE/AGA/00000/base/tokens.xml from zip file
167 ->unzip_xml('GOE/AGA/00000/base/tokens.xml')
168 ->attr_is('spanList span:nth-child(1)', 'to', 8)
169 ->attr_is('spanList span#t_1', 'from', 9)
170 ->attr_is('spanList span#t_1', 'to', 11)
171 ->attr_is('spanList span#t_67', 'from', 427)
172 ->attr_is('spanList span#t_67', 'to', 430)
173 ->attr_is('spanList span#t_214', 'from', 1209)
174 ->attr_is('spanList span#t_214', 'to', 1212)
175 ->element_count_is('spanList span', 227);
176};
Akroneac374d2020-07-07 09:00:44 +0200177
Marc Kupietzeed4cb12021-02-17 19:39:32 +0100178subtest 'Check KorAP tokenizer for infinite loop bug' => sub {
179
180 my $file = catfile($f, 'data', 'korap_tokenizer_challenge.xml');
181
182 eval {
183 require KorAP::XML::TEI::Tokenizer::KorAP;
184 1;
185 } or do {
186 plan skip_all => "KorAP::XML::TEI::Tokenizer::KorAP cannot be used";
187 };
188
189 test_tei2korapxml(
190 file => $file,
191 param => "-tk -s",
192 tmp => 'script_bug_check'
193 )
194 ->stderr_like(qr!tei2korapxml: .*? text_id=WDD19_H0039\.87242!)
195 ->file_readable('WDD19/H0039/87242/struct/structure.xml');
196};
197
Marc Kupietz985da0c2021-02-15 19:29:50 +0100198subtest 'Sentence split with KorAP tokenizer' => sub {
199
200 eval {
201 require KorAP::XML::TEI::Tokenizer::KorAP;
202 1;
203 } or do {
204 plan skip_all => "KorAP::XML::TEI::Tokenizer::KorAP cannot be used";
205 };
206
207 test_tei2korapxml(
208 file => $file,
209 param => "-tk -s",
210 tmp => 'script_sentence_split'
211 )
212 ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
213 ->file_readable('GOE/AGA/00000/struct/structure.xml')
214 ->unzip_xml('GOE/AGA/00000/struct/structure.xml')
215 ->text_is('span#s25 fs f', 's')
216 ->attr_is('span#s25', 'l', -1)
217 ->attr_is('span#s25', 'to', 54)
218 ->text_is('span#s30 fs f', 's')
219 ->attr_is('span#s30', 'l', -1)
220 ->attr_is('span#s30', 'from', 1099)
221 ->attr_is('span#s30', 'to', 1266);
222};
Akroneac374d2020-07-07 09:00:44 +0200223
Akron044bd922020-07-30 09:12:28 +0200224subtest 'Test Tokenizations' => sub {
Akroneac374d2020-07-07 09:00:44 +0200225
Akron044bd922020-07-30 09:12:28 +0200226 my $t = test_tei2korapxml(
227 file => catfile($f, 'data', 'text_with_blanks.i5.xml'),
228 tmp => 'script_out3',
229 param => '-ti'
230 )->stderr_like(qr!tei2korapxml: .*? text_id=CORP_DOC.00001!);
Akron8b511f92020-07-09 17:28:08 +0200231
Akron044bd922020-07-30 09:12:28 +0200232 # ~ test conservative tokenization ~
233 $t->unzip_xml('CORP/DOC/00001/base/tokens_conservative.xml')
234 ->attr_is('spanList span:nth-child(1)', 'to', 6)
Akron8b511f92020-07-09 17:28:08 +0200235
Akron044bd922020-07-30 09:12:28 +0200236 ->attr_is('spanList span#t_1', 'from', 7)
237 ->attr_is('spanList span#t_1', 'to', 9)
Akron8b511f92020-07-09 17:28:08 +0200238
Akron044bd922020-07-30 09:12:28 +0200239 ->attr_is('spanList span#t_3', 'from', 12)
240 ->attr_is('spanList span#t_3', 'to', 16)
Akron8b511f92020-07-09 17:28:08 +0200241
Akron044bd922020-07-30 09:12:28 +0200242 ->attr_is('spanList span#t_9', 'from', 36)
243 ->attr_is('spanList span#t_9', 'to', 37)
Akron8b511f92020-07-09 17:28:08 +0200244
Akron044bd922020-07-30 09:12:28 +0200245 ->attr_is('spanList span#t_13', 'from', 44)
246 ->attr_is('spanList span#t_13', 'to', 45) # "
Akron8b511f92020-07-09 17:28:08 +0200247
Akron044bd922020-07-30 09:12:28 +0200248 ->attr_is('spanList span#t_14', 'from', 45) # twenty-two
249 ->attr_is('spanList span#t_14', 'to', 55)
Peter Harders42e18a62020-07-21 02:43:26 +0200250
Akron044bd922020-07-30 09:12:28 +0200251 ->attr_is('spanList span#t_15', 'from', 55) # "
252 ->attr_is('spanList span#t_15', 'to', 56)
Akron8b511f92020-07-09 17:28:08 +0200253
Akron044bd922020-07-30 09:12:28 +0200254 ->attr_is('spanList span#t_19', 'from', 66)
255 ->attr_is('spanList span#t_19', 'to', 67)
Peter Hardersb1227172020-07-21 02:12:10 +0200256
Akron044bd922020-07-30 09:12:28 +0200257 ->element_count_is('spanList span', 20);
Peter Harders71f072b2020-07-15 14:15:01 +0200258
Peter Harders71f072b2020-07-15 14:15:01 +0200259
Akron044bd922020-07-30 09:12:28 +0200260 # ~ test aggressive tokenization ~
261 $t->unzip_xml('CORP/DOC/00001/base/tokens_aggressive.xml')
262 ->attr_is('spanList span:nth-child(1)', 'to', 6)
Peter Harders71f072b2020-07-15 14:15:01 +0200263
Akron044bd922020-07-30 09:12:28 +0200264 ->attr_is('spanList span#t_1', 'from', 7)
265 ->attr_is('spanList span#t_1', 'to', 9)
Peter Harders71f072b2020-07-15 14:15:01 +0200266
Akron044bd922020-07-30 09:12:28 +0200267 ->attr_is('spanList span#t_3', 'from', 12)
268 ->attr_is('spanList span#t_3', 'to', 16)
Peter Harders71f072b2020-07-15 14:15:01 +0200269
Akron044bd922020-07-30 09:12:28 +0200270 ->attr_is('spanList span#t_9', 'from', 36)
271 ->attr_is('spanList span#t_9', 'to', 37)
Peter Harders71f072b2020-07-15 14:15:01 +0200272
Akron044bd922020-07-30 09:12:28 +0200273 ->attr_is('spanList span#t_13', 'from', 44)
274 ->attr_is('spanList span#t_13', 'to', 45) # "
Peter Harders71f072b2020-07-15 14:15:01 +0200275
Akron044bd922020-07-30 09:12:28 +0200276 ->attr_is('spanList span#t_14', 'from', 45) # twenty
277 ->attr_is('spanList span#t_14', 'to', 51)
Peter Harders71f072b2020-07-15 14:15:01 +0200278
Akron044bd922020-07-30 09:12:28 +0200279 ->attr_is('spanList span#t_15', 'from', 51) # -
280 ->attr_is('spanList span#t_15', 'to', 52)
Peter Harders71f072b2020-07-15 14:15:01 +0200281
Akron044bd922020-07-30 09:12:28 +0200282 ->attr_is('spanList span#t_16', 'from', 52) # two
283 ->attr_is('spanList span#t_16', 'to', 55)
Peter Harders71f072b2020-07-15 14:15:01 +0200284
Akron044bd922020-07-30 09:12:28 +0200285 ->attr_is('spanList span#t_17', 'from', 55) # "
286 ->attr_is('spanList span#t_17', 'to', 56)
Peter Harders71f072b2020-07-15 14:15:01 +0200287
Akron044bd922020-07-30 09:12:28 +0200288 ->attr_is('spanList span#t_21', 'from', 66)
289 ->attr_is('spanList span#t_21', 'to', 67)
Peter Harders71f072b2020-07-15 14:15:01 +0200290
Akron044bd922020-07-30 09:12:28 +0200291 ->element_count_is('spanList span', 22);
292};
Peter Harders71f072b2020-07-15 14:15:01 +0200293
294
Peter Harders42e18a62020-07-21 02:43:26 +0200295subtest 'Check Tokenization Flags' => sub {
296
297 # Get external tokenizer
298 my $f = dirname(__FILE__);
299 my $cmd = catfile($f, 'cmd', 'tokenizer.pl');
300
301 # Load example file
Akron044bd922020-07-30 09:12:28 +0200302 test_tei2korapxml(
303 file => catfile($f, 'data', 'goe_sample.i5.xml'),
304 param => "-ti -tc 'perl $cmd'",
305 tmp => 'script_tokflags'
306 )
307 ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
308 ->file_exists('GOE/AGA/00000/base/tokens_aggressive.xml')
309 ->file_exists('GOE/AGA/00000/base/tokens_conservative.xml')
310 ->file_exists('GOE/AGA/00000/base/tokens.xml')
311 ;
Peter Harders42e18a62020-07-21 02:43:26 +0200312};
313
Peter Harders1c5ce152020-07-22 18:02:50 +0200314
315subtest 'Test utf-8 handling' => sub {
Peter Harders1c5ce152020-07-22 18:02:50 +0200316 # Introduce invalid utf-8 characters
317 my $text_sigle;
Akron54e363c2020-07-29 11:01:03 +0200318 {
319 no warnings;
320 # $text_sigle printed to file, without encoding: Aþƒ¿¿¿¿¿A_Bþƒ¿¿¿¿¿B.Cþƒ¿¿¿¿¿C
321 # the utf8-sequence 'þƒ¿¿¿¿¿' encodes 32 bit of data (see 0x7FFF_FFFF in perlunicode)
322 $text_sigle = "A\x{FFFF_FFFF}A_B\x{FFFF_FFFF}B.C\x{FFFF_FFFF}C"
323 }
324 # If CHECK is 0, encoding and decoding replace any malformed character
325 # with a substitution character.
Peter Harders1c5ce152020-07-22 18:02:50 +0200326 # � = substitution character
327 my $text_sigle_lax = encode_utf8($text_sigle);
328 my $text_sigle_esc = encode('UTF-8', $text_sigle);
329
330 is(length($text_sigle), 11); # A�A_B�B.C�C (char string => length(�) = 1)
331 is(length($text_sigle_lax), 29); # Aþƒ¿¿¿¿¿A_Bþƒ¿¿¿¿¿B.Cþƒ¿¿¿¿¿C (byte string)
332 is(length($text_sigle_esc), 17); # A�A_B�B.C�C (byte string => length(�) = 3)
333
Akron54e363c2020-07-29 11:01:03 +0200334
335 my $tpl;
336 {
337 no warnings;
338 $tpl = i5_template(
339 korpusSigle => "A\x{FFFF_FFFF}A",
340 dokumentSigle => "A\x{FFFF_FFFF}A_B\x{FFFF_FFFF}B",
341 text => "<p>d\x{FFFF_FFFF}d e\x{FFFF_FFFF}e f\x{FFFF_FFFF}f</p>",
342 textSigle => $text_sigle
343 );
344 };
Peter Harders1c5ce152020-07-22 18:02:50 +0200345
346 my ($fh, $tplfile) = korap_tempfile('script_out4');
347 binmode($fh);
348 print $fh encode_utf8($tpl); # => text_id=Aþƒ¿¿¿¿¿A_Bþƒ¿¿¿¿¿B.Cþƒ¿¿¿¿¿C
349 close($fh);
350
351 my (undef, $outzip) = korap_tempfile('script_out5');
352
Akron54e363c2020-07-29 11:01:03 +0200353 # because output 'textid=...' goes to STDERR (see script/tei2korapxml)
Akroneaa96232020-10-15 17:06:15 +0200354 binmode STDERR;
Peter Harders1c5ce152020-07-22 18:02:50 +0200355
356 stderr_like(
357 sub { `cat '$tplfile' | perl '$script' -ti > '$outzip'` },
Akroneaa96232020-10-15 17:06:15 +0200358 qr!tei2korapxml: .*? text_id=$text_sigle_esc!, # see above: print $fh encode_utf8($tpl);
Peter Harders1c5ce152020-07-22 18:02:50 +0200359 );
360};
361
Akrone68ec0c2020-07-28 18:06:19 +0200362
Akrondd0be8f2021-02-18 19:29:41 +0100363subtest 'Check structure parsing with defined foundry and folder' => sub {
364 # Load example file
365 my $file = catfile($f, 'data', 'goe_sample.i5.xml');
366
367 my $t = test_tei2korapxml(
368 tmp => 'script_out',
369 file => $file,
370 param => '-ti --inline-structures=myfoundry#mystr'
371 )->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
372 ->file_exists_not('GOE/AGA/00000/struct/structure.xml', 'Structure not generated')
373 ->unzip_xml('GOE/AGA/00000/myfoundry/mystr.xml')
374 ->text_is('span[id=s3] *[name=type]', 'Autobiographie', 'text content')
375 ->text_is('#s3 *[name=type]', 'Autobiographie', 'text content')
376 ->attr_is('#s0','to','1266')
377 ->attr_is('#s0','l','1')
378 ->attr_is('#s18','from','925')
379 ->attr_is('#s18','to','1266')
380 ->attr_is('#s18','l','5')
381 ;
382
383 $t = test_tei2korapxml(
384 tmp => 'script_out',
385 file => $file,
386 param => '-ti --inline-structures=myfoundry'
387 )->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
388 ->file_exists_not('GOE/AGA/00000/struct/structure.xml', 'Structure not generated')
389 ->unzip_xml('GOE/AGA/00000/myfoundry/structure.xml')
390 ->text_is('span[id=s3] *[name=type]', 'Autobiographie', 'text content')
391 ->text_is('#s3 *[name=type]', 'Autobiographie', 'text content')
392 ->attr_is('#s0','to','1266')
393 ->attr_is('#s0','l','1')
394 ->attr_is('#s18','from','925')
395 ->attr_is('#s18','to','1266')
396 ->attr_is('#s18','l','5')
397 ;
398};
399
400
Akrone68ec0c2020-07-28 18:06:19 +0200401subtest 'Check Inline annotations' => sub {
402
403 # Load example file
404 my $file = catfile($f, 'data', 'goe_sample_tagged.i5.xml');
405
Akron044bd922020-07-30 09:12:28 +0200406 my $t = test_tei2korapxml(
407 file => $file,
408 env => 'KORAPXMLTEI_INLINE=1',
409 tmp => 'script_tagged'
410 )
411 ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
Akrone68ec0c2020-07-28 18:06:19 +0200412
Akron044bd922020-07-30 09:12:28 +0200413 # Check zip using xml loy
414 ->unzip_xml('GOE/AGA/00000/tokens/morpho.xml')
Akrone68ec0c2020-07-28 18:06:19 +0200415
Akron044bd922020-07-30 09:12:28 +0200416 ->attr_is('layer', 'docid', 'GOE_AGA.00000')
Akrone68ec0c2020-07-28 18:06:19 +0200417 ->attr_is('spanList span:nth-child(1)', 'id', 's0')
418 ->attr_is('spanList span:nth-child(1)', 'from', '75')
419 ->attr_is('spanList span:nth-child(1)', 'to', '81')
420 ->attr_is('spanList span:nth-child(1)', 'l', '7')
421
422 ->attr_is('span#s0 > fs', 'type', 'lex')
423 ->attr_is('span#s0 > fs', 'xmlns', 'http://www.tei-c.org/ns/1.0')
424 ->attr_is('span#s0 > fs > f > fs > f:nth-child(1)', 'name', 'pos')
425 ->text_is('span#s0 > fs > f > fs > f:nth-child(1)', 'A')
426 ->attr_is('span#s0 > fs > f > fs > f:nth-child(2)', 'name', 'msd')
427 ->text_is('span#s0 > fs > f > fs > f:nth-child(2)', '@NH')
428
429 ->attr_is('span#s25', 'from', '259')
430 ->attr_is('span#s25', 'to', '263')
431 ->attr_is('span#s25', 'l', '7')
432 ->attr_is('span#s25 > fs > f > fs > f:nth-child(1)', 'name', 'pos')
433 ->text_is('span#s25 > fs > f > fs > f:nth-child(1)', 'PRON')
434 ->attr_is('span#s25 > fs > f > fs > f:nth-child(2)', 'name', 'msd')
435 ->text_is('span#s25 > fs > f > fs > f:nth-child(2)', '@NH')
436
437 ->attr_is('span#s58', 'from', '495')
438 ->attr_is('span#s58', 'to', '500')
439 ->attr_is('span#s58', 'l', '7')
440 ->attr_is('span#s58 > fs > f > fs > f:nth-child(1)', 'name', 'pos')
441 ->text_is('span#s58 > fs > f > fs > f:nth-child(1)', 'N')
442 ->attr_is('span#s58 > fs > f > fs > f:nth-child(2)', 'name', 'msd')
443 ->text_is('span#s58 > fs > f > fs > f:nth-child(2)', '@NH')
444
445 ->attr_is('span#s119', 'from', '914')
446 ->attr_is('span#s119', 'to', '925')
447 ->attr_is('span#s119', 'l', '7')
448 ->attr_is('span#s119 > fs > f > fs > f:nth-child(1)', 'name', 'pos')
449 ->text_is('span#s119 > fs > f > fs > f:nth-child(1)', 'A')
450 ->attr_is('span#s119 > fs > f > fs > f:nth-child(2)', 'name', 'msd')
451 ->text_is('span#s119 > fs > f > fs > f:nth-child(2)', '@NH')
452 ->element_exists_not('span#s120')
453 ;
454};
455
Akron1a5271a2021-02-18 13:18:15 +0100456subtest 'Check Inline annotations with defined foundry and folder' => sub {
457 # Load example file
458 my $file = catfile($f, 'data', 'goe_sample_tagged.i5.xml');
459
460 my $t = test_tei2korapxml(
461 file => $file,
462 env => 'KORAPXMLTEI_INLINE=1',
463 tmp => 'script_tagged',
464 param => '--inline-tokens=myfoundry#myfile'
465 )
466 ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
467
468 ->file_exists_not('GOE/AGA/00000/tokens/morpho.xml', 'Morpho not generated')
469
470 # Check zip using xml loy
471 ->unzip_xml('GOE/AGA/00000/myfoundry/myfile.xml')
472
473 ->attr_is('layer', 'docid', 'GOE_AGA.00000')
474 ->attr_is('spanList span:nth-child(1)', 'id', 's0')
475 ->attr_is('spanList span:nth-child(1)', 'from', '75')
476 ->attr_is('spanList span:nth-child(1)', 'to', '81')
477 ->attr_is('spanList span:nth-child(1)', 'l', '7')
478 ;
479
480 $t = test_tei2korapxml(
481 file => $file,
482 env => 'KORAPXMLTEI_INLINE=1',
483 tmp => 'script_tagged',
484 param => '--inline-tokens=myfoundry'
485 )
486 ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
487
488 ->file_exists_not('GOE/AGA/00000/tokens/morpho.xml', 'Morpho not generated')
489
490 # Check zip using xml loy
491 ->unzip_xml('GOE/AGA/00000/myfoundry/morpho.xml')
492
493 ->attr_is('layer', 'docid', 'GOE_AGA.00000')
494 ->attr_is('spanList span:nth-child(1)', 'id', 's0')
495 ->attr_is('spanList span:nth-child(1)', 'from', '75')
496 ->attr_is('spanList span:nth-child(1)', 'to', '81')
497 ->attr_is('spanList span:nth-child(1)', 'l', '7')
498 ;
499};
500
Akron09e0b2c2020-07-28 15:57:01 +0200501subtest 'Check Inline annotations with untagged file' => sub {
502
503 # Load example file
504 my $file = catfile($f, 'data', 'goe_sample.i5.xml');
505
506 my ($fh, $outzip) = korap_tempfile('script_untagged');
507
508 # Generate zip file (unportable!)
509 stderr_like(
510 sub { `cat '$file' | KORAPXMLTEI_INLINE=1 perl '$script' > '$outzip'` },
511 qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!,
512 'Processing 1'
513 );
514
515 # TODO: there should be a better way to test this
516 stderr_unlike(
517 sub { `cat '$file' | KORAPXMLTEI_INLINE=1 perl '$script' > '$outzip'` },
518 qr!.*undefined value.*!,
519 'Processing 2'
520 );
521 #
522
523 ok(-e $outzip, "File $outzip exists");
524
525 my $zip = IO::Uncompress::Unzip->new(
526 $outzip,
527 Name => 'GOE/AGA/00000/tokens/morpho.xml'
528 );
529 ok((not $zip), 'missing morpho.xml');
530
531 $zip = IO::Uncompress::Unzip->new(
532 $outzip,
533 Name => 'GOE/AGA/00000/struct/structure.xml'
534 );
535 ok($zip, 'found structure.xml');
536};
Akrone68ec0c2020-07-28 18:06:19 +0200537
Akron3378dfd2020-08-01 15:01:36 +0200538
Akroneaa96232020-10-15 17:06:15 +0200539subtest 'Check input encoding' => sub {
540
541 # Load example file
542 test_tei2korapxml(
543 file => catfile($f, 'data', 'goe_sample.i5.xml'),
544 env => 'KORAPXMLTEI_INLINE=1',
545 tmp => 'script_utf8_enc'
546 )
547 ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
548 ->unzip_xml('GOE/AGA/00000/data.xml')
549 ->content_like(qr/\Q&quot;Kriegstheater&quot;\E/)
550 ->content_like(qr/\QTür&#39;\E/)
551 ;
552
553 test_tei2korapxml(
554 file => catfile($f, 'data', 'goe_sample.i5.iso.xml'),
555 env => 'KORAPXMLTEI_INLINE=1',
556 tmp => 'script_iso_enc'
557 )
558 ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
559 ->unzip_xml('GOE/AGA/00000/data.xml')
560 ->content_like(qr/\Q&quot;Kriegstheater&quot;\E/)
561 ->content_like(qr/\QTür&#39;\E/)
562 ;
563};
564
Marc Kupietz44b1f252020-11-26 16:31:40 +0100565subtest 'Check encoding with utf-8 sigle' => sub {
566
567 # Load example file
568 my $file = catfile($f, 'data', 'wdd_sample.i5.xml');
569
570 my $t = test_tei2korapxml(
571 tmp => 'script_sigle',
572 file => $file,
573 param => "-ti"
574 )->stderr_like(qr!tei2korapxml: .*? text_id=WDD19_ß0000\.10317!)
575 ->stderr_unlike(qr!Debugging is activated!);
576
577 $t->unzip_xml('WDD19/ß0000/10317/header.xml')
578 ->text_is('idsHeader fileDesc titleStmt textSigle', 'WDD19/ß0000.10317');
579
580 $t->unzip_xml('WDD19/ß0000/10317/data.xml')
581 ->attr_is('raw_text', 'docid', 'WDD19_ß0000.10317');
582
583 $t->unzip_xml('WDD19/ß0000/10317/struct/structure.xml')
584 ->attr_is('layer', 'docid', 'WDD19_ß0000.10317');
585
586 $t->unzip_xml('WDD19/ß0000/10317/base/tokens_conservative.xml')
587 ->attr_is('layer', 'docid', 'WDD19_ß0000.10317');
588};
589
Marc Kupietz8a954e52021-02-16 22:03:07 +0100590subtest 'Check entity replacement' => sub {
591 my $t = test_tei2korapxml(
592 file => catfile($f, 'data', 'text_with_entities.i5.xml'),
593 tmp => 'script_entity_replacement',
594 param => '-ti'
595 )->stderr_like(qr!tei2korapxml: .*? text_id=CORP_DOC.00003!);
596
597 $t->unzip_xml('CORP/DOC/00003/data.xml')
598 ->content_like(qr!üüü Aα≈„▒░▓█╗┐┌╔═─┬╦┴╩╝┘└╚│║┼╬┤╣╠├•ˇčˆ†‡ě€ƒ…‗ıι“„▄‹‘‚—–νœŒωΩ‰φπϖř”ρ›’‘šŠσ□■▪⊂˜™▀ŸžŽ!);
599
600 $t->unzip_xml('CORP/DOC/00003/header.xml')
601 ->content_like(qr!üüü x α•α y!);
602};
603
Akron3378dfd2020-08-01 15:01:36 +0200604subtest 'Test Log' => sub {
605 test_tei2korapxml(
606 tmp => 'script_out',
607 file => $file,
608 param => '-l=warn'
609 )->stderr_is('');
610};
611
612
Akron797e8072020-02-13 07:59:40 +0100613done_testing;