Fix end of text behaviour in case of sentence positions
Change-Id: Ic433dd3579d9a79df5734a405e682596c3ccddad
diff --git a/token_writer_test.go b/token_writer_test.go
index 868e69d..63b9c2b 100644
--- a/token_writer_test.go
+++ b/token_writer_test.go
@@ -85,6 +85,18 @@
matStr = w.String()
assert.Equal("1 5 5 6\n1 6\n0 3 3 4\n0 4\n", matStr)
+ w.Reset()
+ mat.TransduceTokenWriter(strings.NewReader("Tree\n\x04\n"), tws)
+
+ matStr = w.String()
+ assert.Equal("0 4\n0 4\n", matStr)
+
+ w.Reset()
+ mat.TransduceTokenWriter(strings.NewReader("Tree.\n\x04\n"), tws)
+
+ matStr = w.String()
+ assert.Equal("0 4 4 5\n0 5\n", matStr)
+
//
// Write sentence offsets without token offsets
tws = NewTokenWriter(w, SENTENCE_POS|NEWLINE_AFTER_EOT)