Fixed gap behind last token and <base/s:t> length
Change-Id: I7b8d9cc90280c29d3ba90a8f97ddb63315dc8b0c
diff --git a/t/real/goethe-2.t b/t/real/goethe-2.t
index ff3cba4..691889d 100644
--- a/t/real/goethe-2.t
+++ b/t/real/goethe-2.t
@@ -138,7 +138,7 @@
like($first_token, qr/s:Autobiographische/, 'data');
like($first_token, qr/_0\$<i>0<i>17/, 'data');
like($first_token, qr!<>:dereko/s:s\$<b>64<i>0<i>30<i>2<b>4!, 'data');
-like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35242<i>5233<b>0!, 'data');
+like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35242<i>5234<b>0!, 'data');
# like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35250<i>5233<b>0!, 'data');
like($first_token, qr!<>:base/s:s\$<b>64<i>0<i>30<i>2<b>2!, 'data');
like($first_token, qr!-:base\/paragraphs\$\<i\>14!, 'data');
diff --git a/t/real/goethe.t b/t/real/goethe.t
index 5bc5eb2..03af7cc 100644
--- a/t/real/goethe.t
+++ b/t/real/goethe.t
@@ -138,7 +138,7 @@
like($first_token, qr/s:Autobiographische/, 'data');
like($first_token, qr/_0\$<i>0<i>17/, 'data');
like($first_token, qr!<>:base/s:s\$<b>64<i>0<i>30<i>2<b>2!, 'data');
-like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35199<i>5226<b>0!, 'data');
+like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35199<i>5227<b>0!, 'data');
## OpenNLP
$tokens->add('OpenNLP', 'Sentences');
diff --git a/t/real/hnc.t b/t/real/hnc.t
index 9aca507..5e44f6b 100644
--- a/t/real/hnc.t
+++ b/t/real/hnc.t
@@ -88,8 +88,9 @@
my $output = decode_json( $tokens->to_json );
-is($output->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>4368<i>577<b>0', 't');
+is($output->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>4368<i>578<b>0', 't');
is($output->{data}->{stream}->[0]->[3], 'i:addbot', 't');
+is($output->{data}->{stream}->[-1]->[0], '_577$<i>4359<i>4368', 't');
## Base
diff --git a/t/real/redew.t b/t/real/redew.t
index 25f369c..a5fe45b 100644
--- a/t/real/redew.t
+++ b/t/real/redew.t
@@ -114,15 +114,15 @@
my $first = $output->{data}->{stream}->[0];
is('-:tokens$<i>13',$first->[0]);
-is('<>:base/s:t$<b>64<i>0<i>197<i>12<b>0',$first->[1]);
-is('<>:dereko/s:text$<b>64<i>0<i>197<i>12<b>0',$first->[2]);
-is('<>:dereko/s:body$<b>64<i>118<i>197<i>12<b>1',$first->[3]);
-is('<>:dereko/s:p$<b>64<i>118<i>197<i>12<b>2',$first->[4]);
-is('<>:dereko/s:said$<b>64<i>118<i>197<i>12<b>3<s>1',$first->[5]);
-is('@:dereko/s:level:1$<b>17<s>1<i>12',$first->[6]);
-is('@:dereko/s:content:speech$<b>17<s>1<i>12',$first->[7]);
-is('@:dereko/s:mode:direct$<b>17<s>1<i>12',$first->[8]);
-is('@:dereko/s:id:1$<b>17<s>1<i>12',$first->[9]);
+is('<>:base/s:t$<b>64<i>0<i>197<i>13<b>0',$first->[1]);
+is('<>:dereko/s:text$<b>64<i>0<i>197<i>13<b>0',$first->[2]);
+is('<>:dereko/s:body$<b>64<i>118<i>197<i>13<b>1',$first->[3]);
+is('<>:dereko/s:p$<b>64<i>118<i>197<i>13<b>2',$first->[4]);
+is('<>:dereko/s:said$<b>64<i>118<i>197<i>13<b>3<s>1',$first->[5]);
+is('@:dereko/s:level:1$<b>17<s>1<i>13',$first->[6]);
+is('@:dereko/s:content:speech$<b>17<s>1<i>13',$first->[7]);
+is('@:dereko/s:mode:direct$<b>17<s>1<i>13',$first->[8]);
+is('@:dereko/s:id:1$<b>17<s>1<i>13',$first->[9]);
is('_0$<i>123<i>128',$first->[10]);
is("drukola/l:H\x{f6}rst",$first->[11]);
is('drukola/m:msd:rfpos',$first->[12]);
@@ -135,11 +135,11 @@
my $nine = join(',', @{$output->{data}->{stream}->[9]});
like($nine, qr{drukola\/l:nichts}, 'Nichts');
like($nine, qr{_9\$<i>170<i>176}, 'Term boundaries');
-unlike($nine, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>12<b>4<s>1}, 'Term boundaries');
+unlike($nine, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>13<b>4<s>1}, 'Term boundaries');
my $ten = join(',', @{$output->{data}->{stream}->[10]});
like($ten, qr{_10\$<i>177<i>180}, 'Term boundaries');
-like($ten, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>12<b>4<s>1}, 'Term boundaries');
+like($ten, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>13<b>4<s>1}, 'Term boundaries');
my $eleven = join(',', @{$output->{data}->{stream}->[11]});
like($eleven, qr{_11\$<i>181<i>188}, 'Term boundaries');
diff --git a/t/real/rei.t b/t/real/rei.t
index d50a732..e08516d 100644
--- a/t/real/rei.t
+++ b/t/real/rei.t
@@ -209,7 +209,7 @@
is('<>:dereko/s:docTitle$<b>64<i>0<i>91<i>11<b>3', $first->[15]);
is('<>:dereko/s:titlePart$<b>64<i>0<i>91<i>11<b>4<s>2', $first->[16]);
is('<>:dereko/s:s$<b>64<i>0<i>91<i>11<b>5', $first->[17]);
-is('<>:base/s:t$<b>64<i>0<i>17859<i>2640<b>0', $first->[18]);
+is('<>:base/s:t$<b>64<i>0<i>17859<i>2641<b>0', $first->[18]);
is('>:malt/d:ROOT$<b>33<i>0<i>48<i>0<i>7', $first->[19]);
is('<:malt/d:PP$<b>32<i>1', $first->[20]);
is('<:malt/d:ROOT$<b>34<i>0<i>48<i>7<i>0', $first->[21]);
@@ -298,8 +298,8 @@
is('<>:dereko/s:titlePage$<b>64<i>0<i>61<i>8<b>2<s>1', $first->[17]);
is('<>:dereko/s:docTitle$<b>64<i>0<i>61<i>8<b>3', $first->[18]);
is('<>:opennlp/s:s$<b>64<i>0<i>173<i>24<b>0', $first->[19]);
-is('<>:base/s:t$<b>64<i>0<i>7008<i>1008<b>0', $first->[20]);
-is('<>:dereko/s:text$<b>64<i>0<i>7008<i>1008<b>0', $first->[21]);
+is('<>:base/s:t$<b>64<i>0<i>7008<i>1009<b>0', $first->[20]);
+is('<>:dereko/s:text$<b>64<i>0<i>7008<i>1009<b>0', $first->[21]);
is('>:malt/d:GMOD$<b>32<i>3', $first->[22]);
is('<:malt/d:ROOT$<b>34<i>0<i>51<i>6<i>3', $first->[23]);
is('@:dereko/s:id:bng.00071-0-titlepage$<b>17<s>1<i>8', $first->[24]);
@@ -313,16 +313,17 @@
is('tt/p:NE', $first->[32]);
$last = $output->{data}->{stream}->[-1];
-is('<>:dereko/s:back$<b>65<i>7008<i>7008<i>1008<b>1', $last->[0]);
-is('<>:dereko/s:div$<b>65<i>7008<i>7008<i>1008<b>2<s>1', $last->[1]);
-is('@:dereko/s:n:1$<b>17<s>1', $last->[2]);
-is('@:dereko/s:type:footnotes$<b>17<s>1', $last->[3]);
-is('@:dereko/s:complete:y$<b>17<s>1', $last->[4]);
-is('_1008$<i>6990<i>7006', $last->[5]);
-is('corenlp/p:NN', $last->[6]);
-is('i:befreiungsschlag', $last->[7]);
-is('opennlp/p:NN', $last->[8]);
-is('s:Befreiungsschlag', $last->[9]);
+# No longer indexed:
+#is('<>:dereko/s:back$<b>65<i>7008<i>7008<i>1009<b>1', $last->[0]);
+#is('<>:dereko/s:div$<b>65<i>7008<i>7008<i>1009<b>2<s>1', $last->[1]);
+#is('@:dereko/s:n:1$<b>17<s>1', $last->[2]);
+#is('@:dereko/s:type:footnotes$<b>17<s>1', $last->[3]);
+#is('@:dereko/s:complete:y$<b>17<s>1', $last->[4]);
+is('_1008$<i>6990<i>7006', $last->[0]);
+is('corenlp/p:NN', $last->[1]);
+is('i:befreiungsschlag', $last->[2]);
+is('opennlp/p:NN', $last->[3]);
+is('s:Befreiungsschlag', $last->[4]);
done_testing;
__END__