Added base/s=t during tokenization (2)
Change-Id: I005be1ef836aa1016219c63c45b4fd2bb0431ffa
diff --git a/t/sgbr/lemma.t b/t/sgbr/lemma.t
index 87f1d83..3221dba 100644
--- a/t/sgbr/lemma.t
+++ b/t/sgbr/lemma.t
@@ -32,11 +32,12 @@
my $stream = $data->{stream};
is($stream->[0]->[0], '-:tokens$<i>51', 'Token number');
-is($stream->[0]->[1], '_0$<i>0<i>18', 'Position');
-is($stream->[0]->[2], 'i:sommerüberraschung', 'First term');
-is($stream->[0]->[3], 's:Sommerüberraschung', 'First term');
-is($stream->[0]->[4], 'sgbr/l:Sommerüberraschung', 'First term');
-ok(!defined $stream->[0]->[5], 'First term');
+is($stream->[0]->[1], '<>:base/s:t$<b>64<i>0<i>365<i>50<b>0', 'Text Boundary');
+is($stream->[0]->[2], '_0$<i>0<i>18', 'Position');
+is($stream->[0]->[3], 'i:sommerüberraschung', 'First term');
+is($stream->[0]->[4], 's:Sommerüberraschung', 'First term');
+is($stream->[0]->[5], 'sgbr/l:Sommerüberraschung', 'First term');
+ok(!defined $stream->[0]->[6], 'First term');
is($stream->[1]->[0], '_1$<i>19<i>21', 'Position');
is($stream->[1]->[1], 'i:es', 'Second term');
diff --git a/t/sgbr/pos.t b/t/sgbr/pos.t
index e87a1d7..0163ed1 100644
--- a/t/sgbr/pos.t
+++ b/t/sgbr/pos.t
@@ -33,10 +33,11 @@
my $stream = $data->{stream};
is($stream->[0]->[0], '-:tokens$<i>51', 'Token number');
-is($stream->[0]->[1], '_0$<i>0<i>18', 'Position');
-is($stream->[0]->[2], 'i:sommerüberraschung', 'First term');
-is($stream->[0]->[3], 's:Sommerüberraschung', 'First term');
-is($stream->[0]->[4], 'sgbr/p:NN', 'First term POS');
+is($stream->[0]->[1], '<>:base/s:t$<b>64<i>0<i>365<i>50<b>0', 'Text boundary');
+is($stream->[0]->[2], '_0$<i>0<i>18', 'Position');
+is($stream->[0]->[3], 'i:sommerüberraschung', 'First term');
+is($stream->[0]->[4], 's:Sommerüberraschung', 'First term');
+is($stream->[0]->[5], 'sgbr/p:NN', 'First term POS');
is($stream->[1]->[3], 'sgbr/p:PPER', 'First term POS');
is($stream->[-1]->[3], 'sgbr/p:NE', 'Last term POS');
diff --git a/t/sgbr/token.t b/t/sgbr/token.t
index cafa3c4..da17f8a 100644
--- a/t/sgbr/token.t
+++ b/t/sgbr/token.t
@@ -31,9 +31,9 @@
my $stream = $data->{stream};
is($stream->[0]->[0], '-:tokens$<i>51', 'Token number');
-is($stream->[0]->[1], '_0$<i>0<i>18', 'Position');
-is($stream->[0]->[2], 'i:sommerüberraschung', 'First term');
-is($stream->[0]->[3], 's:Sommerüberraschung', 'First term');
+is($stream->[0]->[2], '_0$<i>0<i>18', 'Position');
+is($stream->[0]->[3], 'i:sommerüberraschung', 'First term');
+is($stream->[0]->[4], 's:Sommerüberraschung', 'First term');
is($stream->[-1]->[0], '_50$<i>359<i>364', 'Last position');
is($stream->[-1]->[1], 'i:kevin', 'Last term');
is($stream->[-1]->[2], 's:Kevin', 'Last term');