Ignore newline after EOT with a flag
Change-Id: Ia18cc0cbb1dda6311c6b2b8db1fae52c4b6335e0
diff --git a/token_writer_test.go b/token_writer_test.go
index 291c3b9..f7bd1f7 100644
--- a/token_writer_test.go
+++ b/token_writer_test.go
@@ -37,7 +37,7 @@
b := make([]byte, 0, 2048)
w := bytes.NewBuffer(b)
- tws := NewTokenWriterFromOptions(w, true)
+ tws := NewTokenWriterFromOptions(w, true, false)
mat := LoadMatrixFile("testdata/tokenizer.matok")
@@ -56,4 +56,12 @@
matStr = w.String()
assert.Equal("This\n.\n\n1 5 5 6\nAnd\n.\n\n1 4 4 5\n", matStr)
+ // Accept newline after EOT
+ tws = NewTokenWriterFromOptions(w, true, true)
+
+ w.Reset()
+ mat.TransduceTokenWriter(strings.NewReader("\nThis.\x0a\x04\nAnd.\n\x04\n"), tws)
+
+ matStr = w.String()
+ assert.Equal("This\n.\n\n1 5 5 6\nAnd\n.\n\n0 3 3 4\n", matStr)
}