measures for multiple sentence counts, improved test suite
diff --git a/t/artificial.t b/t/artificial.t
new file mode 100644
index 0000000..7d40177
--- /dev/null
+++ b/t/artificial.t
@@ -0,0 +1,288 @@
+#!/usr/bin/env perl
+# source ~/perl5/perlbrew/etc/bashrc
+# perlbrew switch perl-blead@korap
+use strict;
+use warnings;
+use utf8;
+use Test::More;
+use Benchmark ':hireswallclock';
+use lib 'lib', '../lib';
+use Scalar::Util qw/weaken/;
+
+use File::Basename 'dirname';
+use File::Spec::Functions 'catdir';
+
+use_ok('KorAP::Document');
+
+my $path = catdir(dirname(__FILE__), 'artificial');
+ok(my $doc = KorAP::Document->new( path => $path . '/' ), 'Load Korap::Document');
+is($doc->path, $path . '/', 'Path');
+ok($doc->parse, 'Parse document');
+
+sub new_tokenizer {
+ my $x = $doc;
+ weaken $x;
+ return KorAP::Tokenizer->new(
+ path => $x->path,
+ doc => $x,
+ foundry => 'OpenNLP',
+ layer => 'Tokens',
+ name => 'tokens'
+ )
+};
+
+is($doc->primary->data,
+ 'Zum letzten kulturellen Anlass lädt die Leitung des Schulheimes Hofbergli ein, '.
+ 'bevor der Betrieb Ende Schuljahr eingestellt wird.', 'Primary data');
+
+is($doc->primary->data_length, 129, 'Primary data length');
+
+is($doc->primary->data(0,3), 'Zum', 'Get primary data');
+
+# Get tokens
+use_ok('KorAP::Tokenizer');
+# Get tokenization
+ok(my $tokens = KorAP::Tokenizer->new(
+ path => $doc->path,
+ doc => $doc,
+ foundry => 'OpenNLP',
+ layer => 'Tokens',
+ name => 'tokens'
+), 'New Tokenizer');
+ok($tokens->parse, 'Parse');
+
+is($tokens->foundry, 'OpenNLP', 'Foundry');
+
+is($tokens->doc->id, 'ART_00001', 'Doc id');
+is($tokens->should, 20, 'Should');
+is($tokens->have, 18, 'Have');
+is($tokens->name, 'tokens', 'Name');
+is($tokens->layer, 'Tokens', 'Layer');
+
+is($tokens->stream->pos(0)->to_string, '[(0-3)s:Zum|i:zum|_0#0-3|-:tokens$<i>18]', 'Token is correct');
+is($tokens->stream->pos(1)->to_string, '[(4-11)s:letzten|i:letzten|_1#4-11]', 'Token is correct');
+
+my $i = 2;
+foreach ([12,23, 'kulturellen'],
+ [24,30, 'Anlass'],
+ [31,35, 'lädt'],
+ [36,39, 'die'],
+ [40,47, 'Leitung'],
+ [48,51, 'des'],
+ [52,63, 'Schulheimes'],
+ [64,73, 'Hofbergli'],
+ [74,77, 'ein'],
+ [79,84, 'bevor'],
+ [85,88, 'der'],
+ [89,96, 'Betrieb'],
+ [97,101, 'Ende'],
+ [102,111, 'Schuljahr'],
+ [112,123, 'eingestellt'],
+ [124,128, 'wird']
+ ) {
+ is($tokens->stream->pos($i++)->to_string,
+ '[('.$_->[0].'-'.$_->[1].')'.
+ 's:'.$_->[2].'|i:'.lc($_->[2]).'|'.
+ '_'.($i-1).'#'.$_->[0].'-'.$_->[1].']',
+ 'Token is correct');
+};
+
+ok(!$tokens->stream->pos($i++), 'No more tokens');
+
+# Add OpenNLP/morpho
+ok($tokens->add('OpenNLP', 'Morpho'), 'Add OpenNLP/Morpho');
+
+$i = 0;
+foreach (qw/APPRART ADJA ADJA NN VVFIN ART NN ART NN NE PTKVZ KOUS ART NN NN NN VVPP VAFIN/) {
+ like($tokens->stream->pos($i++)->to_string,
+ qr!\|opennlp/p:$_!,
+ 'Annotation (OpenNLP) is correct');
+};
+
+# Add OpenNLP/sentences
+ok($tokens->add('OpenNLP', 'Sentences'), 'Add OpenNLP/Sentences');
+
+is($tokens->stream->pos(0)->to_string, '[(0-3)s:Zum|i:zum|_0#0-3|-:tokens$<i>18|opennlp/p:APPRART|<>:opennlp/s#0-129$<i>17|-:opennlp/sentences$<i>1]', 'Correct sentence');
+
+
+# New instantiation
+ok($tokens = KorAP::Tokenizer->new(
+ path => $doc->path,
+ doc => $doc,
+ foundry => 'OpenNLP',
+ layer => 'Tokens',
+ name => 'tokens'
+), 'New Tokenizer');
+
+ok($tokens->parse, 'Parse');
+
+# Add OpenNLP/sentences
+ok($tokens->add('Base', 'Sentences'), 'Add Base/Sentences');
+
+# Add OpenNLP/sentences
+ok($tokens->add('Base', 'Paragraphs'), 'Add Base/Paragraphs');
+
+is($tokens->stream->pos(0)->to_string,
+ '[(0-3)s:Zum|i:zum|_0#0-3|-:tokens$<i>18|<>:base/s#0-129$<i>17|<>:base/text#0-129$<i>17|-:base/sentences$<i>1|-:base/paragraphs$<i>0]',
+ 'Correct base annotation');
+
+
+# New instantiation
+ok($tokens = new_tokenizer, 'New Tokenizer');
+
+ok($tokens->parse, 'Parse');
+
+# Add CoreNLP/NamedEntities
+ok($tokens->add('CoreNLP', 'NamedEntities', 'ne_dewac_175m_600'), 'Add CoreNLP/NamedEntities');
+ok($tokens->add('CoreNLP', 'NamedEntities', 'ne_hgc_175m_600'), 'Add CoreNLP/NamedEntities');
+
+is($tokens->stream->pos(9)->to_string,
+ '[(64-73)s:Hofbergli|i:hofbergli|_9#64-73|corenlp/ne_dewac_175m_600:I-LOC|corenlp/ne_hgc_175m_600:I-LOC]',
+ 'Correct NamedEntities annotation');
+
+
+# New instantiation
+ok($tokens = new_tokenizer, 'New Tokenizer');
+ok($tokens->parse, 'Parse');
+
+# Add CoreNLP/Morpho
+ok($tokens->add('CoreNLP', 'Morpho'), 'Add CoreNLP/Morpho');
+
+is($tokens->stream->pos(0)->to_string,
+ '[(0-3)s:Zum|i:zum|_0#0-3|-:tokens$<i>18|corenlp/p:APPRART]',
+ 'Correct corenlp annotation');
+
+$i = 0;
+foreach (qw/APPRART ADJ ADJA NN VVFIN ART NN ART NN NE PTKVZ KOUS ART NN NN NN VVPP VAFIN/) {
+ like($tokens->stream->pos($i++)->to_string,
+ qr!\|corenlp/p:$_!,
+ 'Annotation (CoreNLP) is correct');
+};
+
+# Add CoreNLP/Sentences
+ok($tokens->add('CoreNLP', 'Sentences'), 'Add CoreNLP/Sentences');
+
+is($tokens->stream->pos(0)->to_string,
+ '[(0-3)s:Zum|i:zum|_0#0-3|-:tokens$<i>18|corenlp/p:APPRART|<>:corenlp/s#0-129$<i>17|-:corenlp/sentences$<i>1]',
+ 'Correct corenlp annotation');
+
+
+# New instantiation
+ok($tokens = new_tokenizer, 'New Tokenizer');
+ok($tokens->parse, 'Parse');
+
+# Add CoreNLP/Sentences
+ok($tokens->add('Connexor', 'Sentences'), 'Add Connexor/Sentences');
+
+is($tokens->stream->pos(0)->to_string,
+ '[(0-3)s:Zum|i:zum|_0#0-3|-:tokens$<i>18|<>:cnx/s#0-129$<i>17|-:cnx/sentences$<i>1]',
+ 'Correct cnx annotation');
+
+
+
+# Todo: CoreNLP/Constituency!
+# Todo: Connexor/Morpho
+# Todo: Connexor/Phrase
+# Todo: Connexor/Syntax
+
+
+done_testing;
+__END__
+
+
+
+# Connexor
+push(@layers, ['Connexor', 'Morpho']);
+push(@layers, ['Connexor', 'Syntax']);
+push(@layers, ['Connexor', 'Phrase']);
+push(@layers, ['Connexor', 'Sentences']);
+
+# TreeTagger
+push(@layers, ['TreeTagger', 'Morpho']);
+push(@layers, ['TreeTagger', 'Sentences']);
+
+# Mate
+# push(@layers, ['Mate', 'Morpho']);
+push(@layers, ['Mate', 'Dependency']);
+
+# XIP
+push(@layers, ['XIP', 'Morpho']);
+push(@layers, ['XIP', 'Constituency']);
+push(@layers, ['XIP', 'Dependency']);
+push(@layers, ['XIP', 'Sentences']);
+
+
+
+# Metdata
+is($doc->title, 'A', 'title');
+ok(!$doc->sub_title, 'subTitle');
+
+is($doc->id, 'WPD_AAA.00001', 'ID');
+is($doc->corpus_id, 'WPD', 'corpusID');
+is($doc->pub_date, '20050328', 'pubDate');
+is($doc->pub_place, 'URL:http://de.wikipedia.org', 'pubPlace');
+is($doc->text_class->[0], 'freizeit-unterhaltung', 'TextClass');
+is($doc->text_class->[1], 'reisen', 'TextClass');
+is($doc->text_class->[2], 'wissenschaft', 'TextClass');
+is($doc->text_class->[3], 'populaerwissenschaft', 'TextClass');
+ok(!$doc->text_class->[4], 'TextClass');
+is($doc->author->[0], 'Ruru', 'author');
+is($doc->author->[1], 'Jens.Ol', 'author');
+is($doc->author->[2], 'Aglarech', 'author');
+ok(!$doc->author->[3], 'author');
+
+# Get tokens
+use_ok('KorAP::Tokenizer');
+# Get tokenization
+ok(my $tokens = KorAP::Tokenizer->new(
+ path => $doc->path,
+ doc => $doc,
+ foundry => 'OpenNLP',
+ layer => 'Tokens',
+ name => 'tokens'
+), 'New Tokenizer');
+ok($tokens->parse, 'Parse');
+
+is($tokens->path, $path . '/', 'Path');
+is($tokens->foundry, 'OpenNLP', 'Foundry');
+is($tokens->doc->id, 'WPD_AAA.00001', 'Doc id');
+is($tokens->should, 1068, 'Should');
+is($tokens->have, 923, 'Have');
+is($tokens->name, 'tokens', 'Name');
+is($tokens->layer, 'Tokens', 'Layer');
+
+is($tokens->stream->pos(118)->to_string, '[(763-768)s:Linie|i:linie|_118#763-768]', 'Token is correct');
+
+# Add Mate
+ok($tokens->add('Mate', 'Morpho'), 'Add Mate');
+
+is($tokens->stream->pos(118)->to_string, '[(763-768)s:Linie|i:linie|_118#763-768|mate/l:linie|mate/p:NN|mate/m:case:acc|mate/m:number:sg|mate/m:gender:fem]', 'with Mate');
+
+# Add sentences
+ok($tokens->add('Base', 'Sentences'), 'Add Sentences');
+
+is($tokens->stream->pos(0)->to_string, '[(0-1)s:A|i:a|_0#0-1|-:tokens$<i>923|mate/p:XY|<>:base/s#0-74$<i>13|<>:base/text#0-6083$<i>923|-:sentences$<i>96]', 'Startinfo');
+
+foreach (@layers) {
+ ok($tokens->add(@$_), 'Add '. join(', ', @$_));
+};
+
+is($tokens->stream->pos(0)->to_string, '[(0-1)s:A|i:a|_0#0-1|-:tokens$<i>923|mate/p:XY|<>:base/s#0-74$<i>13|<>:base/text#0-6083$<i>923|-:sentences$<i>96|<>:base/para#0-224$<i>34|-:paragraphs$<i>76|opennlp/p:NE|<>:opennlp/s#0-74$<i>13|<>:corenlp/s#0-6$<i>2|cnx/l:A|cnx/p:N|cnx/syn:@NH|<>:cnx/s#0-74$<i>13|tt/l:A|tt/p:NN|tt/l:A|tt/p:FM|<>:tt/s#0-6083$<i>923|>:mate/d:PNC$<i>2|xip/p:SYMBOL|xip/l:A|<>:xip/c:TOP#0-74$<i>13|<>:xip/c:MC#0-73$<i>13<b>1|>:xip/d:SUBJ$<i>3|<:xip/d:COORD$<i>1|<>:xip/s#0-74$<i>13]', 'Startinfo');
+
+
+is($tokens->stream->pos(118)->to_string,
+ '[(763-768)s:Linie|i:linie|_118#763-768|'.
+ 'mate/l:linie|mate/p:NN|mate/m:case:acc|mate/m:number:sg|mate/m:gender:fem|' .
+ 'opennlp/p:NN|'.
+ 'cnx/l:linie|cnx/p:N|cnx/syn:@NH|'.
+ 'tt/l:Linie|tt/p:NN|'.
+ '<:mate/d:NK$<i>116|<:mate/d:NK$<i>117|>:mate/d:NK$<i>115|'.
+ 'xip/p:NOUN|xip/l:Linie|<>:xip/c:NOUN#763-768$<i>119|<:xip/d:DETERM$<i>116|<:xip/d:NMOD$<i>117]', 'with All');
+
+is($tokens->layer_info, 'cnx/c=const cnx/l=lemma cnx/m=msd cnx/p=pos mate/d=dep mate/l=lemma mate/m=msd mate/p=pos opennlp/p=pos tt/l=lemma tt/p=pos xip/c=const xip/d=dep xip/l=lemma xip/p=pos', 'Layer info');
+
+is($tokens->support, 'base base/paragraphs base/sentences connexor connexor/morpho connexor/phrase connexor/sentences connexor/syntax corenlp corenlp/namedentities corenlp/namedentities corenlp/namedentities/ne_dewac_175m_600 corenlp/namedentities/ne_hgc_175m_600 corenlp/sentences mate mate/dependency mate/morpho opennlp opennlp/morpho opennlp/sentences treetagger treetagger/morpho treetagger/sentences xip xip/constituency xip/dependency xip/morpho xip/sentences', 'Support');
+
+done_testing;
+
+__END__