Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 1 | #/usr/bin/env perl |
| 2 | use strict; |
| 3 | use warnings; |
| 4 | use File::Basename 'dirname'; |
| 5 | use File::Spec::Functions qw/catdir catfile/; |
Akron | d5bb434 | 2017-06-19 11:50:49 +0200 | [diff] [blame] | 6 | use File::Temp qw/:POSIX/; |
Akron | 3ec0a1c | 2017-01-18 14:41:55 +0100 | [diff] [blame] | 7 | use Mojo::File; |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 8 | use Mojo::JSON qw/decode_json/; |
| 9 | use IO::Uncompress::Gunzip; |
| 10 | use Test::More; |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 11 | use Test::Output; |
| 12 | use Data::Dumper; |
Akron | f98b669 | 2016-08-16 19:17:44 +0200 | [diff] [blame] | 13 | use utf8; |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 14 | |
| 15 | my $f = dirname(__FILE__); |
| 16 | my $script = catfile($f, '..', '..', 'script', 'korapxml2krill'); |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 17 | |
| 18 | my $input = catdir($f, '..', 'annotation', 'corpus', 'doc', '0001'); |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 19 | ok(-d $input, 'Input directory found'); |
| 20 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 21 | my $output = tmpnam(); |
Akron | d5bb434 | 2017-06-19 11:50:49 +0200 | [diff] [blame] | 22 | my $cache = tmpnam(); |
| 23 | |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 24 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 25 | ok(!-f $output, 'Output does not exist'); |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 26 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 27 | my $call = join( |
| 28 | ' ', |
| 29 | 'perl', $script, |
| 30 | '--input' => $input, |
| 31 | '--output' => $output, |
Akron | d5bb434 | 2017-06-19 11:50:49 +0200 | [diff] [blame] | 32 | '--cache' => $cache, |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 33 | '-t' => 'OpenNLP#Tokens', |
| 34 | '-l' => 'INFO' |
| 35 | ); |
| 36 | |
| 37 | # Test without compression |
| 38 | stderr_like( |
| 39 | sub { |
| 40 | system($call); |
| 41 | }, |
| 42 | qr!The code took!, |
| 43 | $call |
| 44 | ); |
| 45 | |
| 46 | ok(-f $output, 'Output does exist'); |
Akron | 3ec0a1c | 2017-01-18 14:41:55 +0100 | [diff] [blame] | 47 | ok((my $file = Mojo::File->new($output)->slurp), 'Slurp data'); |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 48 | ok((my $json = decode_json $file), 'decode json'); |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 49 | is($json->{textType}, 'Zeitung: Tageszeitung', 'text type'); |
| 50 | is($json->{title}, 'Beispiel Text', 'Title'); |
| 51 | is($json->{data}->{tokenSource}, 'opennlp#tokens', 'Title'); |
| 52 | is($json->{data}->{foundries}, 'base base/paragraphs base/sentences connexor connexor/morpho connexor/phrase connexor/sentences connexor/syntax corenlp corenlp/constituency corenlp/morpho corenlp/sentences dereko dereko/structure glemm glemm/morpho mate mate/dependency mate/morpho opennlp opennlp/morpho opennlp/sentences treetagger treetagger/morpho treetagger/sentences xip xip/constituency xip/morpho xip/sentences', 'Foundries'); |
| 53 | like($json->{data}->{text}, qr/^Zum letzten kulturellen/, 'Foundries'); |
| 54 | is($json->{data}->{stream}->[0]->[0], '-:base/paragraphs$<i>1', 'Paragraphs'); |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 55 | is($json->{data}->{tokenSource}, 'opennlp#tokens', 'TokenSource'); |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 56 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 57 | # Delete output |
| 58 | unlink $output; |
| 59 | ok(!-f $output, 'Output does not exist'); |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 60 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 61 | $call .= ' -z'; |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 62 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 63 | # Test with compression |
| 64 | stderr_like( |
| 65 | sub { system($call); }, |
| 66 | qr!The code took!, |
| 67 | $call |
| 68 | ); |
| 69 | |
| 70 | ok(-f $output, 'Output does exist'); |
| 71 | |
| 72 | # Uncompress the data using a buffer |
| 73 | my $gz = IO::Uncompress::Gunzip->new($output, Transparent => 0); |
| 74 | ($file, my $buffer) = ''; |
| 75 | while ($gz->read($buffer)) { |
| 76 | $file .= $buffer; |
| 77 | }; |
Nils Diewald | b3e9ccd | 2016-10-24 15:16:52 +0200 | [diff] [blame] | 78 | $gz->close; |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 79 | |
| 80 | ok($json = decode_json($file), 'decode json'); |
| 81 | |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 82 | is($json->{textType}, 'Zeitung: Tageszeitung', 'text type'); |
| 83 | is($json->{title}, 'Beispiel Text', 'Title'); |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 84 | is($json->{data}->{tokenSource}, 'opennlp#tokens', 'TokenSource'); |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 85 | is($json->{data}->{foundries}, 'base base/paragraphs base/sentences connexor connexor/morpho connexor/phrase connexor/sentences connexor/syntax corenlp corenlp/constituency corenlp/morpho corenlp/sentences dereko dereko/structure glemm glemm/morpho mate mate/dependency mate/morpho opennlp opennlp/morpho opennlp/sentences treetagger treetagger/morpho treetagger/sentences xip xip/constituency xip/morpho xip/sentences', 'Foundries'); |
| 86 | like($json->{data}->{text}, qr/^Zum letzten kulturellen/, 'Foundries'); |
| 87 | is($json->{data}->{stream}->[0]->[0], '-:base/paragraphs$<i>1', 'Paragraphs'); |
| 88 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 89 | # Delete output |
Nils Diewald | b3e9ccd | 2016-10-24 15:16:52 +0200 | [diff] [blame] | 90 | is(unlink($output), 1, 'Unlink successful'); |
| 91 | ok(!-e $output, 'Output does not exist'); |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 92 | |
| 93 | # Use a different token source and skip all annotations, |
| 94 | # except for DeReKo#Structure and Mate#Dependency |
| 95 | $call = join( |
| 96 | ' ', |
| 97 | 'perl', $script, |
| 98 | '--input' => $input, |
| 99 | '--output' => $output, |
Akron | d5bb434 | 2017-06-19 11:50:49 +0200 | [diff] [blame] | 100 | '--cache' => $cache, |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 101 | '-t' => 'CoreNLP#Tokens', |
| 102 | '-s' => '#all', |
| 103 | '-a' => 'DeReKo#Structure', |
| 104 | '-a' => 'Mate#Dependency', |
| 105 | '-l' => 'INFO' |
| 106 | ); |
| 107 | |
| 108 | stderr_like( |
| 109 | sub { |
| 110 | system($call); |
| 111 | }, |
| 112 | qr!The code took!, |
| 113 | $call |
| 114 | ); |
| 115 | |
| 116 | ok(-f $output, 'Output does exist'); |
Akron | 3ec0a1c | 2017-01-18 14:41:55 +0100 | [diff] [blame] | 117 | ok(($file = Mojo::File->new($output)->slurp), 'Slurp data'); |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 118 | ok(($json = decode_json $file), 'decode json'); |
| 119 | |
| 120 | is($json->{textType}, 'Zeitung: Tageszeitung', 'text type'); |
| 121 | |
| 122 | is($json->{title}, 'Beispiel Text', 'Title'); |
| 123 | is($json->{data}->{tokenSource}, 'corenlp#tokens', 'TokenSource'); |
| 124 | is($json->{data}->{foundries}, 'dereko dereko/structure mate mate/dependency', 'Foundries'); |
| 125 | |
| 126 | like($json->{data}->{text}, qr/^Zum letzten kulturellen/, 'Foundries'); |
| 127 | is($json->{data}->{stream}->[0]->[0], '-:tokens$<i>20', 'Tokens'); |
| 128 | |
Akron | e2b902d | 2016-08-16 16:50:11 +0200 | [diff] [blame] | 129 | |
| 130 | # Check overwrite |
| 131 | $call = join( |
| 132 | ' ', |
| 133 | 'perl', $script, |
| 134 | '--input' => $input, |
| 135 | '--output' => $output, |
Akron | d5bb434 | 2017-06-19 11:50:49 +0200 | [diff] [blame] | 136 | '--cache' => $cache, |
Akron | e2b902d | 2016-08-16 16:50:11 +0200 | [diff] [blame] | 137 | '-t' => 'CoreNLP#Tokens', |
| 138 | '-s' => '#all', |
| 139 | '-a' => 'DeReKo#Structure', |
| 140 | '-a' => 'Mate#Dependency', |
| 141 | '-l' => 'DEBUG' |
| 142 | ); |
| 143 | |
| 144 | ok(-f $output, 'Output does exist'); |
| 145 | stderr_like( |
| 146 | sub { |
| 147 | system($call); |
| 148 | }, |
| 149 | qr!already exists!, |
| 150 | $call |
| 151 | ); |
| 152 | |
| 153 | $call .= ' -w '; |
| 154 | |
| 155 | stderr_unlike( |
| 156 | sub { |
| 157 | system($call); |
| 158 | }, |
| 159 | qr!already exists!, |
| 160 | $call |
| 161 | ); |
| 162 | |
Akron | f98b669 | 2016-08-16 19:17:44 +0200 | [diff] [blame] | 163 | # Check meta data switch |
Akron | e2b902d | 2016-08-16 16:50:11 +0200 | [diff] [blame] | 164 | |
Akron | f98b669 | 2016-08-16 19:17:44 +0200 | [diff] [blame] | 165 | # Delete output |
| 166 | unlink $output; |
| 167 | ok(!-f $output, 'Output does not exist'); |
| 168 | |
| 169 | $input = catdir($f, '..', 'sgbr', 'PRO-DUD', 'BSP-2013-01', '32'); |
| 170 | |
| 171 | # Use a different token source and skip all annotations, |
| 172 | # except for DeReKo#Structure and Mate#Dependency |
| 173 | $call = join( |
| 174 | ' ', |
| 175 | 'perl', $script, |
| 176 | '--input' => $input, |
| 177 | '--output' => $output, |
Akron | d5bb434 | 2017-06-19 11:50:49 +0200 | [diff] [blame] | 178 | '--cache' => $cache, |
Akron | f98b669 | 2016-08-16 19:17:44 +0200 | [diff] [blame] | 179 | '-m' => 'Sgbr', |
| 180 | '-t' => 'Base#Tokens_aggr', |
| 181 | '-l' => 'INFO' |
| 182 | ); |
| 183 | |
| 184 | stderr_like( |
| 185 | sub { |
| 186 | system($call); |
| 187 | }, |
| 188 | qr!The code took!, |
| 189 | $call |
| 190 | ); |
| 191 | |
| 192 | ok(-f $output, 'Output does exist'); |
Akron | 3ec0a1c | 2017-01-18 14:41:55 +0100 | [diff] [blame] | 193 | ok(($file = Mojo::File->new($output)->slurp), 'Slurp data'); |
Akron | f98b669 | 2016-08-16 19:17:44 +0200 | [diff] [blame] | 194 | ok(($json = decode_json $file), 'decode json'); |
| 195 | |
| 196 | is($json->{data}->{text}, 'Selbst ist der Jeck', 'Text'); |
| 197 | is($json->{data}->{tokenSource}, 'base#tokens_aggr', 'TokenSource'); |
| 198 | is($json->{pubPlace}, 'Stadtingen', 'pubPlace'); |
| 199 | is($json->{textSigle}, 'PRO-DUD/BSP-2013-01/32', 'textSigle'); |
| 200 | is($json->{docSigle}, 'PRO-DUD/BSP-2013-01', 'docSigle'); |
| 201 | is($json->{corpusSigle}, 'PRO-DUD', 'corpusSigle'); |
| 202 | is($json->{sgbrKodex}, 'T', 'sgbrKodex'); |
| 203 | is($json->{author}, 'unbekannt', 'Author'); |
| 204 | is($json->{language}, 'de', 'Language'); |
| 205 | is($json->{docTitle}, 'Korpus zur Beobachtung des Schreibgebrauchs im Deutschen', 'docTitle'); |
| 206 | is($json->{funder}, 'Bundesministerium für Bildung und Forschung', 'docTitle'); |
| 207 | is($json->{title}, 'Nur Platt, kein Deutsch', 'title'); |
| 208 | is($json->{pubDate}, '20130126', 'pubDate'); |
| 209 | is($json->{docSubTitle}, 'Subkorpus Ortsblatt, Jahrgang 2013, Monat Januar', 'docSubTitle'); |
| 210 | is($json->{keywords}, 'sgbrKodex:T', 'keywords'); |
| 211 | is($json->{publisher}, 'Dorfblatt GmbH', 'publisher'); |
| 212 | |
Akron | 636bd9c | 2017-02-09 17:13:00 +0100 | [diff] [blame] | 213 | |
| 214 | |
| 215 | # AGA with base info |
| 216 | unlink $output; |
| 217 | ok(!-f $output, 'Output does not exist'); |
| 218 | $input = catdir($f, '..', 'corpus', 'GOE2', 'AGA', '03828'); |
| 219 | ok(-d $input, 'Input directory found'); |
| 220 | |
| 221 | ok(!-f $output, 'Output does not exist'); |
| 222 | |
| 223 | $call = join( |
| 224 | ' ', |
| 225 | 'perl', $script, |
| 226 | '--input' => $input, |
| 227 | '--output' => $output, |
Akron | d5bb434 | 2017-06-19 11:50:49 +0200 | [diff] [blame] | 228 | '--cache' => $cache, |
Akron | 636bd9c | 2017-02-09 17:13:00 +0100 | [diff] [blame] | 229 | '-t' => 'base#tokens_aggr', |
| 230 | '-bs' => 'DeReKo#Structure', |
| 231 | '-bp' => 'DeReKo#Structure', |
| 232 | '-bpb' => 'DeReKo#Structure', |
| 233 | '-l' => 'INFO' |
| 234 | ); |
| 235 | |
| 236 | stderr_like( |
| 237 | sub { |
| 238 | system($call); |
| 239 | }, |
| 240 | qr!The code took!, |
| 241 | $call |
| 242 | ); |
| 243 | ok(-f $output, 'Output does exist'); |
| 244 | ok(($file = Mojo::File->new($output)->slurp), 'Slurp data'); |
| 245 | ok(($json = decode_json $file), 'decode json'); |
| 246 | |
| 247 | is($json->{title}, 'Autobiographische Einzelheiten', 'title'); |
| 248 | is($json->{data}->{stream}->[0]->[-1], '~:base/s:pb$<i>529<i>0', 'Pagebreak annotation'); |
| 249 | |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 250 | done_testing; |
| 251 | __END__ |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 252 | |