Support token offsets in token writer
Change-Id: I7db20d8f26fd87a2f43c3a6599bfeba050fc340a
diff --git a/token_writer_test.go b/token_writer_test.go
index 8ab6ed0..71524fb 100644
--- a/token_writer_test.go
+++ b/token_writer_test.go
@@ -2,6 +2,7 @@
import (
"bytes"
+ "strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -13,7 +14,7 @@
b := make([]byte, 0, 2048)
w := bytes.NewBuffer(b)
- tws := NewTokenWriterSimple(w)
+ tws := NewTokenWriter(w)
assert.NotNil(tws)
@@ -29,3 +30,23 @@
assert.Equal("abc\nef\n\n\n", w.String())
}
+
+func TestTokenWriterFromOptions(t *testing.T) {
+ assert := assert.New(t)
+
+ b := make([]byte, 0, 2048)
+ w := bytes.NewBuffer(b)
+
+ tws := NewTokenWriterFromOptions(w, true)
+
+ mat := LoadMatrixFile("testdata/tokenizer.matok")
+
+ assert.NotNil(mat)
+
+ assert.True(mat.TransduceTokenWriter(
+ strings.NewReader("This.\x0a\x04And.\n\x04\n"), tws),
+ )
+
+ matStr := w.String()
+ assert.Equal("This\n.\n\n0 4 4 5\nAnd\n.\n\n0 3 3 4\n", matStr)
+}