Added range test and prepare fixing of relations
diff --git a/t/artificial.t b/t/artificial.t
index 92ebb8a..95ef890 100644
--- a/t/artificial.t
+++ b/t/artificial.t
@@ -14,6 +14,18 @@
use_ok('KorAP::Document');
+# Tests for material identicality of a token
+sub _t2h {
+ my $string = shift;
+ $string =~ s/^\[\(\d+?-\d+?\)(.+?)\]$/$1/;
+ my %hash = ();
+ foreach (split(qr!\|!, $string)) {
+ $hash{$_} = 1;
+ };
+ return \%hash;
+};
+
+
my $path = catdir(dirname(__FILE__), 'artificial');
ok(my $doc = KorAP::Document->new( path => $path . '/' ), 'Load Korap::Document');
like($doc->path, qr!$path/$!, 'Path');
@@ -93,7 +105,6 @@
# Add OpenNLP/morpho
ok($tokens->add('OpenNLP', 'Morpho'), 'Add OpenNLP/Morpho');
-
$i = 0;
foreach (qw/APPRART ADJA ADJA NN VVFIN ART NN ART NN NE PTKVZ KOUS ART NN NN NN VVPP VAFIN/) {
like($tokens->stream->pos($i++)->to_string,
@@ -107,7 +118,7 @@
is($tokens->stream->pos(0)->to_string,
'[(0-3)-:opennlp/sentences$<i>1|-:tokens$<i>18|<>:opennlp/s:s#0-129$<i>17<b>0|_0#0-3|i:zum|opennlp/p:APPRART|s:Zum]',
-# '[(0-3)-:opennlp/sentences$<i>1|-:tokens$<i>18|_0#0-3|i:zum|s:Zum|opennlp/p:APPRART|<>:opennlp/s:s#0-129$<i>17]',
+ # '[(0-3)-:opennlp/sentences$<i>1|-:tokens$<i>18|_0#0-3|i:zum|s:Zum|opennlp/p:APPRART|<>:opennlp/s:s#0-129$<i>17]',
'Correct sentence'
);
@@ -128,9 +139,9 @@
# Add OpenNLP/sentences
ok($tokens->add('Base', 'Paragraphs'), 'Add Base/Paragraphs');
-is($tokens->stream->pos(0)->to_string,
- '[(0-3)-:base/paragraphs$<i>0|-:base/sentences$<i>1|-:tokens$<i>18|<>:base/s:t#0-129$<i>17<b>0|<>:base/s:s#0-129$<i>17<b>2|_0#0-3|i:zum|s:Zum]',
-# '[(0-3)-:base/paragraphs$<i>0|-:base/sentences$<i>1|-:tokens$<i>18|_0#0-3|i:zum|s:Zum|<>:base/s:t#0-129$<i>17<b>0|<>:base/s:s#0-129$<i>17<b>0]',
+is_deeply(
+ _t2h($tokens->stream->pos(0)->to_string),
+ _t2h('[(0-3)-:base/paragraphs$<i>1|-:base/sentences$<i>1|-:tokens$<i>18|<>:base/s:t#0-129$<i>17<b>0|<>:base/s:p#0-129$<i>17<b>1|<>:base/s:s#0-129$<i>17<b>2|_0#0-3|i:zum|s:Zum]'),
'Correct base annotation');
# New instantiation
@@ -141,9 +152,11 @@
ok($tokens->add('CoreNLP', 'NamedEntities', 'ne_hgc_175m_600'), 'Add CoreNLP/NamedEntities');
# [(64-73)s:Hofbergli|i:hofbergli|_9#64-73|corenlp/ne_dewac_175m_600:I-LOC|corenlp/ne_hgc_175m_600:I-LOC]
-is($tokens->stream->pos(9)->to_string,
- '[(64-73)_9#64-73|corenlp/ne:I-LOC|i:hofbergli|s:Hofbergli]',
- 'Correct NamedEntities annotation');
+is_deeply(
+ _t2h($tokens->stream->pos(9)->to_string),
+ _t2h('[(64-73)_9#64-73|corenlp/ne:I-LOC|i:hofbergli|s:Hofbergli]'),
+ 'Correct NamedEntities annotation'
+);
# New instantiation
ok($tokens = new_tokenizer->parse, 'Parse');
@@ -151,10 +164,11 @@
# Add CoreNLP/Morpho
ok($tokens->add('CoreNLP', 'Morpho'), 'Add CoreNLP/Morpho');
-is($tokens->stream->pos(0)->to_string,
- '[(0-3)-:tokens$<i>18|_0#0-3|corenlp/p:APPRART|i:zum|s:Zum]',
-# '[(0-3)-:tokens$<i>18|_0#0-3|i:zum|s:Zum|corenlp/p:APPRART]',
- 'Correct corenlp annotation');
+is_deeply(
+ _t2h($tokens->stream->pos(0)->to_string),
+ _t2h('[(0-3)-:tokens$<i>18|_0#0-3|corenlp/p:APPRART|i:zum|s:Zum]'),
+ 'Correct corenlp annotation'
+);
$i = 0;
foreach (qw/APPRART ADJ ADJA NN VVFIN ART NN ART NN NE PTKVZ KOUS ART NN NN NN VVPP VAFIN/) {
@@ -164,15 +178,15 @@
};
-
# Add CoreNLP/Sentences
ok($tokens->add('CoreNLP', 'Sentences'), 'Add CoreNLP/Sentences');
-is($tokens->stream->pos(0)->to_string,
- '[(0-3)-:corenlp/sentences$<i>1|-:tokens$<i>18|<>:corenlp/s:s#0-129$<i>17<b>0|_0#0-3|corenlp/p:APPRART|i:zum|s:Zum]',
-# '[(0-3)-:corenlp/sentences$<i>1|-:tokens$<i>18|_0#0-3|i:zum|s:Zum|corenlp/p:APPRART|<>:corenlp/s:s#0-129$<i>17]',
- 'Correct corenlp annotation');
-
+is_deeply(
+ _t2h($tokens->stream->pos(0)->to_string),
+ _t2h('[(0-3)-:corenlp/sentences$<i>1|-:tokens$<i>18|<>:corenlp/s:s#0-129$<i>17<b>0|_0#0-3|corenlp/p:APPRART|i:zum|s:Zum]'),
+ # '[(0-3)-:corenlp/sentences$<i>1|-:tokens$<i>18|_0#0-3|i:zum|s:Zum|corenlp/p:APPRART|<>:corenlp/s:s#0-129$<i>17]',
+ 'Correct corenlp annotation'
+);
# New instantiation
ok($tokens = new_tokenizer->parse, 'New Tokenizer');
@@ -180,10 +194,12 @@
# Add CoreNLP/Sentences
ok($tokens->add('Connexor', 'Sentences'), 'Add Connexor/Sentences');
-is($tokens->stream->pos(0)->to_string,
- '[(0-3)-:cnx/sentences$<i>1|-:tokens$<i>18|<>:cnx/s:s#0-129$<i>17<b>0|_0#0-3|i:zum|s:Zum]',
- # '[(0-3)-:cnx/sentences$<i>1|-:tokens$<i>18|_0#0-3|i:zum|s:Zum|<>:cnx/s:s#0-129$<i>17<b>0]',
- 'Correct cnx annotation');
+is_deeply(
+ _t2h($tokens->stream->pos(0)->to_string),
+ _t2h('[(0-3)-:cnx/sentences$<i>1|-:tokens$<i>18|<>:cnx/s:s#0-129$<i>17<b>0|_0#0-3|i:zum|s:Zum]'),
+ # '[(0-3)-:cnx/sentences$<i>1|-:tokens$<i>18|_0#0-3|i:zum|s:Zum|<>:cnx/s:s#0-129$<i>17<b>0]',
+ 'Correct cnx annotation'
+);
# New instantiation
ok($tokens = new_tokenizer->parse, 'New Tokenizer');
@@ -253,11 +269,12 @@
# Add XIP/Sentences
ok($tokens->add('XIP', 'Sentences'), 'Add XIP/Sentences');
-is($tokens->stream->pos(0)->to_string,
- '[(0-3)-:tokens$<i>18|-:xip/sentences$<i>1|<>:xip/s:s#0-129$<i>17<b>0|_0#0-3|i:zum|s:Zum]',
-# '[(0-3)-:tokens$<i>18|_0#0-3|i:zum|s:Zum|-:xip/sentences$<i>1|<>:xip/s:s#0-129$<i>17<b>0]',
- 'First sentence'
- );
+is_deeply(
+ _t2h($tokens->stream->pos(0)->to_string),
+ _t2h('[(0-3)-:tokens$<i>18|-:xip/sentences$<i>1|<>:xip/s:s#0-129$<i>17<b>0|_0#0-3|i:zum|s:Zum]'),
+ # '[(0-3)-:tokens$<i>18|_0#0-3|i:zum|s:Zum|-:xip/sentences$<i>1|<>:xip/s:s#0-129$<i>17<b>0]',
+ 'First sentence'
+);
# Add XIP/Morpho
ok($tokens->add('XIP', 'Morpho'), 'Add XIP/Morpho');
@@ -294,10 +311,16 @@
# Add XIP/Sentences
ok($tokens->add('XIP', 'Dependency'), 'Add XIP/Dependency');
-
$stream = $tokens->stream;
-like($stream->pos(1)->to_string, qr!\|>:xip/d:NMOD\$<i>3!, 'Dependency fine');
-like($stream->pos(3)->to_string, qr!\|<:xip/d:NMOD\$<i>1!, 'Dependency fine');
+diag $stream->pos(1)->to_string;
+
+like($stream->pos(1)->to_string, qr![^<]>:xip/d:NMOD\$<i>3!, 'Dependency fine');
+like($stream->pos(3)->to_string, qr![^<]<:xip/d:NMOD\$<i>1!, 'Dependency fine');
+
+done_testing;
+__END__
+
+
like($stream->pos(3)->to_string, qr!\|<:xip/d:NMOD\$<i>2!, 'Dependency fine');
like($stream->pos(4)->to_string, qr!\|>xip/d:VMAIN\$<i>4!, 'Dependency fine');
like($stream->pos(4)->to_string, qr!\|<:xip/d:SUBJ\$<i>6!, 'Dependency fine');