Introduce test object and simplified script testing
Change-Id: I1f9b31b256b705e3ce54c797098790a07749f6f2
diff --git a/t/script.t b/t/script.t
index 742b417..533e08e 100644
--- a/t/script.t
+++ b/t/script.t
@@ -3,18 +3,16 @@
use File::Basename 'dirname';
use File::Spec::Functions qw/catfile/;
use Encode qw!encode_utf8 decode_utf8 encode!;
-use IO::Uncompress::Unzip qw(unzip $UnzipError);
use Test::More;
use Test::Output;
-use Test::XML::Loy;
use FindBin;
BEGIN {
unshift @INC, "$FindBin::Bin/../lib";
};
-use Test::KorAP::XML::TEI qw!korap_tempfile i5_template!;
+use Test::KorAP::XML::TEI qw!korap_tempfile i5_template test_tei2korapxml!;
my $f = dirname(__FILE__);
my $script = catfile($f, '..', 'script', 'tei2korapxml');
@@ -26,292 +24,183 @@
'Help'
);
+
stdout_like(
sub { system('perl', $script, '--version') },
qr!tei2korapxml - v\d+?\.\d+?!,
'Version'
);
-
# Load example file
my $file = catfile($f, 'data', 'goe_sample.i5.xml');
-my ($fh, $outzip) = korap_tempfile('script_out');
+subtest 'Basic processing' => sub {
-# Generate zip file (unportable!)
-stderr_like(
- sub { `cat '$file' | perl '$script' -ti > '$outzip'` },
-# approaches for working with $fh (also better use OO interface then)
-# sub { open STDOUT, '>&', $fh; system("cat '$file' | perl '$script'") },
-# sub { open(my $pipe, "cat '$file' | perl '$script'|"); while(<$pipe>){$fh->print($_)}; $fh->close },
-# sub {
-# defined(my $pid = fork) or die "fork: $!";
-# if (!$pid) {
-# open STDOUT, '>&', $fh;
-# exec "cat '$file' | perl '$script'"
-# }
-# waitpid $pid, 0;
-# $fh->close;
-# },
- qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!,
- 'Processing'
-);
-
-ok(-e $outzip, "File $outzip exists");
-
-# Uncompress GOE/header.xml from zip file
-my $zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/header.xml');
-
-ok($zip, 'Zip-File is created');
-
-# TODO: check wrong encoding in header-files (compare with input document)!
-# Read GOE/header.xml
-my $header_xml = '';
-$header_xml .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
-
-my $t = Test::XML::Loy->new($header_xml);
-
-$t->text_is('korpusSigle', 'GOE', 'korpusSigle')
- ->text_is('h\.title[type=main]', 'Goethes Werke', 'h.title')
- ->text_is('h\.author', 'Goethe, Johann Wolfgang von', 'h.author')
- ->text_is('pubDate[type=year]', '1982', 'pubDate');
+ my $t = test_tei2korapxml(
+ tmp => 'script_out',
+ file => $file,
+ param => '-ti'
+ )->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!);
-# Uncompress GOE/AGA/header.xml from zip file
-$zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/header.xml');
+ # Uncompress GOE/header.xml from zip file
+ $t->unzip_xml('GOE/header.xml')
-ok($zip, 'Zip-File is found');
+ # TODO: check wrong encoding in header-files (compare with input document)!
+ ->text_is('korpusSigle', 'GOE', 'korpusSigle')
+ ->text_is('h\.title[type=main]', 'Goethes Werke', 'h.title')
+ ->text_is('h\.author', 'Goethe, Johann Wolfgang von', 'h.author')
+ ->text_is('pubDate[type=year]', '1982', 'pubDate');
-# Read GOE/AGA/header.xml
-$header_xml = '';
-$header_xml .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
+ # Uncompress GOE/AGA/header.xml from zip file
+ $t->unzip_xml('GOE/AGA/header.xml')
+ ->text_is('dokumentSigle', 'GOE/AGA', 'dokumentSigle')
+ ->text_is('d\.title', 'Goethe: Autobiographische Schriften II, (1817-1825, 1832)', 'd.title')
+ ->text_is('creatDate', '1820-1822', 'creatDate');
-$t = Test::XML::Loy->new($header_xml);
-
-$t->text_is('dokumentSigle', 'GOE/AGA', 'dokumentSigle')
- ->text_is('d\.title', 'Goethe: Autobiographische Schriften II, (1817-1825, 1832)', 'd.title')
- ->text_is('creatDate', '1820-1822', 'creatDate');
-
-# Uncompress GOE/AGA/00000/header.xml from zip file
-$zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/header.xml');
-
-ok($zip, 'Zip-File is found');
-
-# Read GOE/AGA/00000/header.xml
-$header_xml = '';
-$header_xml .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
-
-$t = Test::XML::Loy->new($header_xml);
-$t->text_is('textSigle', 'GOE/AGA.00000', 'textSigle')
- ->text_is('analytic > h\.title[type=main]', 'Campagne in Frankreich', 'h.title');
+ # Uncompress GOE/AGA/00000/header.xml from zip file
+ $t->unzip_xml('GOE/AGA/00000/header.xml')
+ ->text_is('textSigle', 'GOE/AGA.00000', 'textSigle')
+ ->text_is('analytic > h\.title[type=main]', 'Campagne in Frankreich', 'h.title');
# Uncompress GOE/AGA/00000/data.xml from zip file
-$zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/data.xml');
+ $t->unzip_xml('GOE/AGA/00000/data.xml')
+ ->attr_is('raw_text', 'docid', 'GOE_AGA.00000', 'text id')
+ ->text_like(
+ 'raw_text > text',
+ qr!^Campagne in Frankreich 1792.*?uns allein begl.*cke\.$!,
+ 'text content');
-ok($zip, 'Zip-File is found');
+ $t->unzip_xml('GOE/AGA/00000/struct/structure.xml')
+ ->text_is('span[id=s3] *[name=type]', 'Autobiographie', 'text content');
-# Read GOE/AGA/00000/data.xml
-my $data_xml = '';
-$data_xml .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
+ $t->file_exists_not('GOE/AGA/00000/base/tokens.xml', 'External not generated');
-$t = Test::XML::Loy->new($data_xml);
-$t->attr_is('raw_text', 'docid', 'GOE_AGA.00000', 'text id')
- ->text_like('raw_text > text', qr!^Campagne in Frankreich 1792.*?uns allein begl.*cke\.$!, 'text content');
+ # Uncompress GOE/AGA/00000/base/tokens_aggressive.xml from zip file
+ $t->unzip_xml('GOE/AGA/00000/base/tokens_aggressive.xml')
+ ->attr_is('spanList span:nth-child(1)', 'to', 8)
+ ->attr_is('spanList span#t_1', 'from', 9)
+ ->attr_is('spanList span#t_1', 'to', 11)
-# Uncompress GOE/AGA/00000/struct/structure.xml from zip file
-$zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/struct/structure.xml');
+ ->attr_is('spanList span#t_67', 'from', 427)
+ ->attr_is('spanList span#t_67', 'to', 430)
-ok($zip, 'Zip-File is found');
+ ->attr_is('spanList span#t_214', 'from', 1209)
+ ->attr_is('spanList span#t_214', 'to', 1212)
-# Read GOE/AGA/00000/struct/structure.xml
-my $struct_xml = '';
-$struct_xml .= $zip->getline while !$zip->eof;
+ ->element_count_is('spanList span', 227);
-ok($zip->close, 'Closed');
+ # Uncompress GOE/AGA/00000/base/tokens_conservative.xml from zip file
+ $t->unzip_xml('GOE/AGA/00000/base/tokens_conservative.xml')
+ ->attr_is('spanList span:nth-child(1)', 'to', 8)
-$t = Test::XML::Loy->new($struct_xml);
-$t->text_is('span[id=s3] *[name=type]', 'Autobiographie', 'text content');
+ ->attr_is('spanList span#t_1', 'from', 9)
+ ->attr_is('spanList span#t_1', 'to', 11)
-$zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/base/tokens.xml');
-ok(!$zip, 'External not generated');
+ ->attr_is('spanList span#t_67', 'from', 427)
+ ->attr_is('spanList span#t_67', 'to', 430)
-# Uncompress GOE/AGA/00000/base/tokens_aggressive.xml from zip file
-$zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/base/tokens_aggressive.xml');
+ ->attr_is('spanList span#t_214', 'from', 1209)
+ ->attr_is('spanList span#t_214', 'to', 1212)
-# Read GOE/AGA/00000/base/tok.xml
-my $tokens_xml = '';
-$tokens_xml .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
+ ->element_count_is('spanList span', 227);
+};
-$t = Test::XML::Loy->new($tokens_xml);
-$t->attr_is('spanList span:nth-child(1)', 'to', 8);
-$t->attr_is('spanList span#t_1', 'from', 9);
-$t->attr_is('spanList span#t_1', 'to', 11);
+subtest 'Tokenize with external tokenizer' => sub {
-$t->attr_is('spanList span#t_67', 'from', 427);
-$t->attr_is('spanList span#t_67', 'to', 430);
+ my $cmd = catfile($f, 'cmd', 'tokenizer.pl');
-$t->attr_is('spanList span#t_214', 'from', 1209);
-$t->attr_is('spanList span#t_214', 'to', 1212);
+ test_tei2korapxml(
+ file => $file,
+ param => "-tc='perl $cmd'",
+ tmp => 'script_out2'
+ )
+ ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
+ ->file_readable('GOE/AGA/00000/base/tokens.xml')
-$t->element_count_is('spanList span', 227);
+ # Uncompress GOE/AGA/00000/base/tokens.xml from zip file
+ ->unzip_xml('GOE/AGA/00000/base/tokens.xml')
+ ->attr_is('spanList span:nth-child(1)', 'to', 8)
+ ->attr_is('spanList span#t_1', 'from', 9)
+ ->attr_is('spanList span#t_1', 'to', 11)
+ ->attr_is('spanList span#t_67', 'from', 427)
+ ->attr_is('spanList span#t_67', 'to', 430)
+ ->attr_is('spanList span#t_214', 'from', 1209)
+ ->attr_is('spanList span#t_214', 'to', 1212)
+ ->element_count_is('spanList span', 227);
+};
-# Uncompress GOE/AGA/00000/base/tokens_conservative.xml from zip file
-$zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/base/tokens_conservative.xml');
+subtest 'Test Tokenizations' => sub {
-$tokens_xml = '';
-$tokens_xml .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
+ my $t = test_tei2korapxml(
+ file => catfile($f, 'data', 'text_with_blanks.i5.xml'),
+ tmp => 'script_out3',
+ param => '-ti'
+ )->stderr_like(qr!tei2korapxml: .*? text_id=CORP_DOC.00001!);
-$t = Test::XML::Loy->new($tokens_xml);
-$t->attr_is('spanList span:nth-child(1)', 'to', 8);
+ # ~ test conservative tokenization ~
+ $t->unzip_xml('CORP/DOC/00001/base/tokens_conservative.xml')
+ ->attr_is('spanList span:nth-child(1)', 'to', 6)
-$t->attr_is('spanList span#t_1', 'from', 9);
-$t->attr_is('spanList span#t_1', 'to', 11);
+ ->attr_is('spanList span#t_1', 'from', 7)
+ ->attr_is('spanList span#t_1', 'to', 9)
-$t->attr_is('spanList span#t_67', 'from', 427);
-$t->attr_is('spanList span#t_67', 'to', 430);
+ ->attr_is('spanList span#t_3', 'from', 12)
+ ->attr_is('spanList span#t_3', 'to', 16)
-$t->attr_is('spanList span#t_214', 'from', 1209);
-$t->attr_is('spanList span#t_214', 'to', 1212);
+ ->attr_is('spanList span#t_9', 'from', 36)
+ ->attr_is('spanList span#t_9', 'to', 37)
-$t->element_count_is('spanList span', 227);
+ ->attr_is('spanList span#t_13', 'from', 44)
+ ->attr_is('spanList span#t_13', 'to', 45) # "
+ ->attr_is('spanList span#t_14', 'from', 45) # twenty-two
+ ->attr_is('spanList span#t_14', 'to', 55)
-# Tokenize with external tokenizer
-my $cmd = catfile($f, 'cmd', 'tokenizer.pl');
+ ->attr_is('spanList span#t_15', 'from', 55) # "
+ ->attr_is('spanList span#t_15', 'to', 56)
-my ($fh2, $outzip2) = korap_tempfile('script_out2');
+ ->attr_is('spanList span#t_19', 'from', 66)
+ ->attr_is('spanList span#t_19', 'to', 67)
-stderr_like(
- sub { `cat '$file' | perl '$script' -tc='perl $cmd' > '$outzip2'` },
- qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!,
- 'Processing'
-);
-
-# Uncompress GOE/AGA/00000/base/tokens.xml from zip file
-$zip = IO::Uncompress::Unzip->new($outzip2, Name => 'GOE/AGA/00000/base/tokens.xml');
-ok($zip, 'Found');
-ok(!$zip->eof, 'Readable');
-
-# Read GOE/AGA/00000/base/tokens.xml
-$tokens_xml = '';
-$tokens_xml .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
-
-$t = Test::XML::Loy->new($tokens_xml);
-$t->attr_is('spanList span:nth-child(1)', 'to', 8);
-
-$t->attr_is('spanList span#t_1', 'from', 9);
-$t->attr_is('spanList span#t_1', 'to', 11);
-
-$t->attr_is('spanList span#t_67', 'from', 427);
-$t->attr_is('spanList span#t_67', 'to', 430);
-
-$t->attr_is('spanList span#t_214', 'from', 1209);
-$t->attr_is('spanList span#t_214', 'to', 1212);
-
-$t->element_count_is('spanList span', 227);
-
-
-my ($fh3, $outzip3) = korap_tempfile('script_out3');
-
-
-# ~ test conservative tokenization ~
-
-$file = catfile($f, 'data', 'text_with_blanks.i5.xml');
-
-stderr_like(
- sub { `cat '$file' | perl '$script' --ti > '$outzip3'` },
- qr!tei2korapxml: .*? text_id=CORP_DOC.00001!,
- 'Processing'
-);
-
-ok(-e $outzip3, "File $outzip3 exists");
-
-$zip = IO::Uncompress::Unzip->new($outzip3, Name => 'CORP/DOC/00001/base/tokens_conservative.xml');
-
-ok($zip, 'Zip-File is created');
-
-my $cons = '';
-$cons .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
-
-$t = Test::XML::Loy->new($cons);
-$t->attr_is('spanList span:nth-child(1)', 'to', 6);
-
-$t->attr_is('spanList span#t_1', 'from', 7);
-$t->attr_is('spanList span#t_1', 'to', 9);
-
-$t->attr_is('spanList span#t_3', 'from', 12);
-$t->attr_is('spanList span#t_3', 'to', 16);
-
-$t->attr_is('spanList span#t_9', 'from', 36);
-$t->attr_is('spanList span#t_9', 'to', 37);
-
-$t->attr_is('spanList span#t_13', 'from', 44);
-$t->attr_is('spanList span#t_13', 'to', 45); # "
-
-$t->attr_is('spanList span#t_14', 'from', 45); # twenty-two
-$t->attr_is('spanList span#t_14', 'to', 55);
-
-$t->attr_is('spanList span#t_15', 'from', 55); # "
-$t->attr_is('spanList span#t_15', 'to', 56);
-
-$t->attr_is('spanList span#t_19', 'from', 66);
-$t->attr_is('spanList span#t_19', 'to', 67);
-
-$t->element_count_is('spanList span', 20);
-
-
-# ~ test aggressive tokenization ~
-
-$zip = IO::Uncompress::Unzip->new($outzip3, Name => 'CORP/DOC/00001/base/tokens_aggressive.xml');
-
-ok($zip, 'Zip-File is created');
-
-my $aggr = '';
-$aggr .= $zip->getline while !$zip->eof;
-ok($zip->close, 'Closed');
+ ->element_count_is('spanList span', 20);
-$t = Test::XML::Loy->new($aggr);
-$t->attr_is('spanList span:nth-child(1)', 'to', 6);
+ # ~ test aggressive tokenization ~
+ $t->unzip_xml('CORP/DOC/00001/base/tokens_aggressive.xml')
+ ->attr_is('spanList span:nth-child(1)', 'to', 6)
-$t->attr_is('spanList span#t_1', 'from', 7);
-$t->attr_is('spanList span#t_1', 'to', 9);
+ ->attr_is('spanList span#t_1', 'from', 7)
+ ->attr_is('spanList span#t_1', 'to', 9)
-$t->attr_is('spanList span#t_3', 'from', 12);
-$t->attr_is('spanList span#t_3', 'to', 16);
+ ->attr_is('spanList span#t_3', 'from', 12)
+ ->attr_is('spanList span#t_3', 'to', 16)
-$t->attr_is('spanList span#t_9', 'from', 36);
-$t->attr_is('spanList span#t_9', 'to', 37);
+ ->attr_is('spanList span#t_9', 'from', 36)
+ ->attr_is('spanList span#t_9', 'to', 37)
-$t->attr_is('spanList span#t_13', 'from', 44);
-$t->attr_is('spanList span#t_13', 'to', 45); # "
+ ->attr_is('spanList span#t_13', 'from', 44)
+ ->attr_is('spanList span#t_13', 'to', 45) # "
-$t->attr_is('spanList span#t_14', 'from', 45); # twenty
-$t->attr_is('spanList span#t_14', 'to', 51);
+ ->attr_is('spanList span#t_14', 'from', 45) # twenty
+ ->attr_is('spanList span#t_14', 'to', 51)
-$t->attr_is('spanList span#t_15', 'from', 51); # -
-$t->attr_is('spanList span#t_15', 'to', 52);
+ ->attr_is('spanList span#t_15', 'from', 51) # -
+ ->attr_is('spanList span#t_15', 'to', 52)
-$t->attr_is('spanList span#t_16', 'from', 52); # two
-$t->attr_is('spanList span#t_16', 'to', 55);
+ ->attr_is('spanList span#t_16', 'from', 52) # two
+ ->attr_is('spanList span#t_16', 'to', 55)
-$t->attr_is('spanList span#t_17', 'from', 55); # "
-$t->attr_is('spanList span#t_17', 'to', 56);
+ ->attr_is('spanList span#t_17', 'from', 55) # "
+ ->attr_is('spanList span#t_17', 'to', 56)
-$t->attr_is('spanList span#t_21', 'from', 66);
-$t->attr_is('spanList span#t_21', 'to', 67);
+ ->attr_is('spanList span#t_21', 'from', 66)
+ ->attr_is('spanList span#t_21', 'to', 67)
-$t->element_count_is('spanList span', 22);
+ ->element_count_is('spanList span', 22);
+};
subtest 'Check Tokenization Flags' => sub {
@@ -321,25 +210,16 @@
my $cmd = catfile($f, 'cmd', 'tokenizer.pl');
# Load example file
- my $file = catfile($f, 'data', 'goe_sample.i5.xml');
-
- my ($fh, $outzip) = korap_tempfile('script_tokflags');
-
- # Generate zip file (unportable!)
- stderr_like(
- sub { `cat '$file' | perl '$script' -ti -tc 'perl $cmd' > '$outzip'` },
- qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!,
- 'Processing'
- );
-
- ok(-e $outzip, "File $outzip exists");
-
- $zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/base/tokens_aggressive.xml');
- ok($zip, 'Aggressive generated');
- $zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/base/tokens_conservative.xml');
- ok($zip, 'Conservative generated');
- $zip = IO::Uncompress::Unzip->new($outzip, Name => 'GOE/AGA/00000/base/tokens.xml');
- ok($zip, 'External generated');
+ test_tei2korapxml(
+ file => catfile($f, 'data', 'goe_sample.i5.xml'),
+ param => "-ti -tc 'perl $cmd'",
+ tmp => 'script_tokflags'
+ )
+ ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
+ ->file_exists('GOE/AGA/00000/base/tokens_aggressive.xml')
+ ->file_exists('GOE/AGA/00000/base/tokens_conservative.xml')
+ ->file_exists('GOE/AGA/00000/base/tokens.xml')
+ ;
};
@@ -396,30 +276,17 @@
# Load example file
my $file = catfile($f, 'data', 'goe_sample_tagged.i5.xml');
- my ($fh, $outzip) = korap_tempfile('script_tagged');
+ my $t = test_tei2korapxml(
+ file => $file,
+ env => 'KORAPXMLTEI_INLINE=1',
+ tmp => 'script_tagged'
+ )
+ ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
- # Generate zip file (unportable!)
- stderr_like(
- sub { `cat '$file' | KORAPXMLTEI_INLINE=1 perl '$script' > '$outzip'` },
- qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!,
- 'Processing'
- );
+ # Check zip using xml loy
+ ->unzip_xml('GOE/AGA/00000/tokens/morpho.xml')
- ok(-e $outzip, "File $outzip exists");
-
- my $zip = IO::Uncompress::Unzip->new(
- $outzip,
- Name => 'GOE/AGA/00000/tokens/morpho.xml'
- );
- ok($zip, 'Inline annotations');
-
- my $tokens;
- $tokens .= $zip->getline while !$zip->eof;
- ok($zip->close, 'Closed');
-
- my $t = Test::XML::Loy->new($tokens);
-
- $t->attr_is('layer', 'docid', 'GOE_AGA.00000')
+ ->attr_is('layer', 'docid', 'GOE_AGA.00000')
->attr_is('spanList span:nth-child(1)', 'id', 's0')
->attr_is('spanList span:nth-child(1)', 'from', '75')
->attr_is('spanList span:nth-child(1)', 'to', '81')