Fixed gap behind last token and <base/s:t> length
Change-Id: I7b8d9cc90280c29d3ba90a8f97ddb63315dc8b0c
diff --git a/t/annotation/base_paragraphs.t b/t/annotation/base_paragraphs.t
index e5d0c6b..9f02f27 100644
--- a/t/annotation/base_paragraphs.t
+++ b/t/annotation/base_paragraphs.t
@@ -17,8 +17,8 @@
like($data->{foundries}, qr!base/paragraphs!, 'data');
is($data->{stream}->[0]->[0], '-:base/paragraphs$<i>1', 'Number of paragraphs');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text');
-is($data->{stream}->[0]->[3], '<>:base/s:p$<b>64<i>0<i>129<i>17<b>1', 'Paragraph');
+is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text');
+is($data->{stream}->[0]->[3], '<>:base/s:p$<b>64<i>0<i>129<i>18<b>1', 'Paragraph');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
done_testing;
diff --git a/t/annotation/base_sentences.t b/t/annotation/base_sentences.t
index 69bb313..8096e5b 100644
--- a/t/annotation/base_sentences.t
+++ b/t/annotation/base_sentences.t
@@ -17,8 +17,8 @@
like($data->{foundries}, qr!base/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:base/sentences$<i>1', 'Number of paragraphs');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text');
-is($data->{stream}->[0]->[3], '<>:base/s:s$<b>64<i>0<i>129<i>17<b>2', 'Sentence');
+is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text');
+is($data->{stream}->[0]->[3], '<>:base/s:s$<b>64<i>0<i>129<i>18<b>2', 'Sentence');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
done_testing;
diff --git a/t/annotation/connexor_morpho.t b/t/annotation/connexor_morpho.t
index de0f704..d426571 100644
--- a/t/annotation/connexor_morpho.t
+++ b/t/annotation/connexor_morpho.t
@@ -14,7 +14,7 @@
my $data = $tokens->to_data->{data};
like($data->{foundries}, qr!connexor/morpho!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[2], '_0$<i>0<i>3', 'Position');
is($data->{stream}->[1]->[1], 'cnx/l:letzt', 'Lemma');
is($data->{stream}->[1]->[2], 'cnx/p:A', 'POS');
diff --git a/t/annotation/connexor_phrase.t b/t/annotation/connexor_phrase.t
index a2cf6e3..f3df759 100644
--- a/t/annotation/connexor_phrase.t
+++ b/t/annotation/connexor_phrase.t
@@ -15,7 +15,7 @@
my $data = $tokens->to_data->{data};
like($data->{foundries}, qr!connexor/phrase!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[1]->[0], '<>:cnx/c:np$<b>64<i>4<i>30<i>4<b>0', 'Noun phrase');
done_testing;
diff --git a/t/annotation/connexor_sentences.t b/t/annotation/connexor_sentences.t
index 0c2891c..37db3fb 100644
--- a/t/annotation/connexor_sentences.t
+++ b/t/annotation/connexor_sentences.t
@@ -42,8 +42,8 @@
like($data->{foundries}, qr!connexor/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:cnx/sentences$<i>1', 'Number of paragraphs');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
-is($data->{stream}->[0]->[3], '<>:cnx/s:s$<b>64<i>0<i>129<i>17<b>0', 'Sentence');
+is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
+is($data->{stream}->[0]->[3], '<>:cnx/s:s$<b>64<i>0<i>129<i>18<b>0', 'Sentence');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
done_testing;
diff --git a/t/annotation/corenlp_morpho.t b/t/annotation/corenlp_morpho.t
index ec6e518..cefc94e 100644
--- a/t/annotation/corenlp_morpho.t
+++ b/t/annotation/corenlp_morpho.t
@@ -15,7 +15,7 @@
my $data = $tokens->to_data->{data};
like($data->{foundries}, qr!corenlp/morpho!, 'data');
like($data->{layerInfos}, qr!corenlp/p=tokens!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[3], 'corenlp/p:APPRART', 'POS');
is($data->{stream}->[1]->[1], 'corenlp/p:ADJ', 'POS');
is($data->{stream}->[2]->[1], 'corenlp/p:ADJA', 'POS');
diff --git a/t/annotation/corenlp_sentences.t b/t/annotation/corenlp_sentences.t
index b9e9b75..09611c6 100644
--- a/t/annotation/corenlp_sentences.t
+++ b/t/annotation/corenlp_sentences.t
@@ -17,8 +17,8 @@
like($data->{foundries}, qr!corenlp/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:corenlp/sentences$<i>1', 'Number of paragraphs');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
-is($data->{stream}->[0]->[3], '<>:corenlp/s:s$<b>64<i>0<i>129<i>17<b>0', 'Text');
+is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
+is($data->{stream}->[0]->[3], '<>:corenlp/s:s$<b>64<i>0<i>129<i>18<b>0', 'Text');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
is($data->{stream}->[-1]->[0], '_17$<i>124<i>128', 'Position');
diff --git a/t/annotation/dereko_struct.t b/t/annotation/dereko_struct.t
index 4e00bb3..921b58e 100644
--- a/t/annotation/dereko_struct.t
+++ b/t/annotation/dereko_struct.t
@@ -23,7 +23,7 @@
'Empty element');
-is($data->{stream}->[0]->[5], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[5], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
# Attributes:
is($data->{stream}->[0]->[11],
diff --git a/t/annotation/glemm_morpho.t b/t/annotation/glemm_morpho.t
index 208d9d1..0ac9705 100644
--- a/t/annotation/glemm_morpho.t
+++ b/t/annotation/glemm_morpho.t
@@ -17,7 +17,7 @@
like($data->{foundries}, qr!glemm/morpho!, 'data');
like($data->{layerInfos}, qr!glemm/l=tokens!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[3], 'glemm/l:__zu', 'Lemma');
is($data->{stream}->[1]->[1], 'glemm/l:__letzt-', 'Lemma');
diff --git a/t/annotation/malt_dependency.t b/t/annotation/malt_dependency.t
index f58204f..515ae8f 100644
--- a/t/annotation/malt_dependency.t
+++ b/t/annotation/malt_dependency.t
@@ -51,7 +51,7 @@
is($stream->[0]->[5], '>:malt/d:ROOT$<b>33<i>0<i>49<i>0<i>6', 'Term2Term relation');
# Text element
-is($stream->[0]->[4], '<>:base/s:t$<b>64<i>0<i>238<i>30<b>0', 'Text element');
+is($stream->[0]->[4], '<>:base/s:t$<b>64<i>0<i>238<i>31<b>0', 'Text element');
done_testing;
__END__
diff --git a/t/annotation/mate_morpho.t b/t/annotation/mate_morpho.t
index 736fd31..9798ba0 100644
--- a/t/annotation/mate_morpho.t
+++ b/t/annotation/mate_morpho.t
@@ -19,7 +19,7 @@
like($data->{layerInfos}, qr!mate/l=tokens!, 'data');
like($data->{layerInfos}, qr!mate/m=tokens!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[4], 'mate/l:zu', 'POS');
is($data->{stream}->[0]->[5], 'mate/m:case:dat', 'POS');
is($data->{stream}->[0]->[6], 'mate/m:gender:neut', 'POS');
diff --git a/t/annotation/mate_morpho_attr.t b/t/annotation/mate_morpho_attr.t
index b32c12c..e556d0f 100644
--- a/t/annotation/mate_morpho_attr.t
+++ b/t/annotation/mate_morpho_attr.t
@@ -18,7 +18,7 @@
like($data->{layerInfos}, qr!mate/p=tokens!, 'data');
like($data->{layerInfos}, qr!mate/l=tokens!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[2], '@:gender=neut$<b>16<s>1', 'POS');
is($data->{stream}->[0]->[3], '@:number=sg$<b>16<s>1', 'POS');
is($data->{stream}->[0]->[4], '@:case=dat$<b>16<s>1', 'POS');
diff --git a/t/annotation/mdp_dependency.t b/t/annotation/mdp_dependency.t
index de9144c..2fd341f 100644
--- a/t/annotation/mdp_dependency.t
+++ b/t/annotation/mdp_dependency.t
@@ -82,8 +82,8 @@
is($stream->[0]->[10], '>:mdp/d:ROOT$<b>33<i>0<i>317<i>0<i>40', 'Term-to-Element');
-is($stream->[-1]->[0], '>:mdp/d:ROOT$<b>33<i>26130<i>26153<i>3553<i>3554', 'Term-to-Element');
-is($stream->[3553]->[1], '<:mdp/d:ROOT$<b>34<i>26130<i>26153<i>3554<i>3553', 'Element-to-Term');
+is($stream->[-1]->[0], '>:mdp/d:ROOT$<b>33<i>26130<i>26153<i>3553<i>3555', 'Term-to-Element');
+is($stream->[3553]->[1], '<:mdp/d:ROOT$<b>34<i>26130<i>26153<i>3555<i>3553', 'Element-to-Term');
done_testing;
__END__
diff --git a/t/annotation/opennlp_morpho.t b/t/annotation/opennlp_morpho.t
index c8bcbb7..64d9008 100644
--- a/t/annotation/opennlp_morpho.t
+++ b/t/annotation/opennlp_morpho.t
@@ -10,7 +10,7 @@
ok(my $tokens = TestInit::tokens('0001'), 'Parse tokens');
-is($tokens->stream->pos(0)->to_string, '[(0-3)-:tokens$<i>18|<>:base/s:t$<b>64<i>0<i>129<i>17<b>0|_0$<i>0<i>3|i:zum|s:Zum]', 'Token is correct');
+is($tokens->stream->pos(0)->to_string, '[(0-3)-:tokens$<i>18|<>:base/s:t$<b>64<i>0<i>129<i>18<b>0|_0$<i>0<i>3|i:zum|s:Zum]', 'Token is correct');
is($tokens->stream->pos(1)->to_string, '[(4-11)_1$<i>4<i>11|i:letzten|s:letzten]', 'Token is correct');
diff --git a/t/annotation/opennlp_sentences.t b/t/annotation/opennlp_sentences.t
index 16c8a65..02eac83 100644
--- a/t/annotation/opennlp_sentences.t
+++ b/t/annotation/opennlp_sentences.t
@@ -17,7 +17,7 @@
like($data->{foundries}, qr!opennlp/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:opennlp/sentences$<i>1', 'Number of Sentences');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[3], '<>:opennlp/s:s$<b>64<i>0<i>129<i>17<b>0', 'Sentence');
+is($data->{stream}->[0]->[3], '<>:opennlp/s:s$<b>64<i>0<i>129<i>18<b>0', 'Sentence');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
done_testing;
diff --git a/t/annotation/tt_sentences.t b/t/annotation/tt_sentences.t
index 703f1a6..a7894b8 100644
--- a/t/annotation/tt_sentences.t
+++ b/t/annotation/tt_sentences.t
@@ -19,7 +19,7 @@
like($data->{foundries}, qr!treetagger/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:tokens$<i>18', 'Number of tokens');
is($data->{stream}->[0]->[1], '-:tt/sentences$<i>1', 'Number of paragraphs');
-is($data->{stream}->[0]->[3], '<>:tt/s:s$<b>64<i>0<i>130<i>17<b>0', 'Text');
+is($data->{stream}->[0]->[3], '<>:tt/s:s$<b>64<i>0<i>130<i>18<b>0', 'Text');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
is($data->{stream}->[-1]->[0], '_17$<i>124<i>128', 'Position');
diff --git a/t/annotation/xip_constituency.t b/t/annotation/xip_constituency.t
index 026321f..0f8a0b8 100644
--- a/t/annotation/xip_constituency.t
+++ b/t/annotation/xip_constituency.t
@@ -19,8 +19,8 @@
# The length includes the punct - but that doesn't matter
is($data->{stream}->[0]->[1], '<>:xip/c:PREP$<b>64<i>0<i>3<i>1<b>3', 'Prep phrase');
is($data->{stream}->[0]->[2], '<>:xip/c:PP$<b>64<i>0<i>30<i>4<b>2', 'pp phrase');
-is($data->{stream}->[0]->[4], '<>:xip/c:TOP$<b>64<i>0<i>129<i>17<b>0', 'top phrase');
-is($data->{stream}->[0]->[5], '<>:xip/c:MC$<b>64<i>0<i>129<i>17<b>1', 'mc phrase');
+is($data->{stream}->[0]->[4], '<>:xip/c:TOP$<b>64<i>0<i>129<i>18<b>0', 'top phrase');
+is($data->{stream}->[0]->[5], '<>:xip/c:MC$<b>64<i>0<i>129<i>18<b>1', 'mc phrase');
is($data->{stream}->[-1]->[0], '<>:xip/c:VERB$<b>64<i>124<i>128<i>18<b>4', 'Noun phrase');
diff --git a/t/annotation/xip_sentences.t b/t/annotation/xip_sentences.t
index 3f2c62c..88b3e21 100644
--- a/t/annotation/xip_sentences.t
+++ b/t/annotation/xip_sentences.t
@@ -18,7 +18,7 @@
is($data->{stream}->[0]->[0], '-:tokens$<i>18', 'Number of tokens');
is($data->{stream}->[0]->[1], '-:xip/sentences$<i>1', 'Number of paragraphs');
-is($data->{stream}->[0]->[3], '<>:xip/s:s$<b>64<i>0<i>129<i>17<b>0', 'Text');
+is($data->{stream}->[0]->[3], '<>:xip/s:s$<b>64<i>0<i>129<i>18<b>0', 'Text');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
is($data->{stream}->[-1]->[0], '_17$<i>124<i>128', 'Position');