Support token offsets in token writer
Change-Id: I7db20d8f26fd87a2f43c3a6599bfeba050fc340a
diff --git a/datok.go b/datok.go
index f8fccaa..f804bf0 100644
--- a/datok.go
+++ b/datok.go
@@ -746,7 +746,7 @@
// Transduce input to ouutput
func (dat *DaTokenizer) Transduce(r io.Reader, w io.Writer) bool {
- return dat.TransduceTokenWriter(r, NewTokenWriterSimple(w))
+ return dat.TransduceTokenWriter(r, NewTokenWriter(w))
}
// TransduceTokenWriter transduces an input string against
@@ -757,7 +757,7 @@
// Based on Mizobuchi et al (2000), p. 129,
// with additional support for IDENTITY, UNKNOWN
// and EPSILON transitions and NONTOKEN and TOKENEND handling.
-func (dat *DaTokenizer) TransduceTokenWriter(r io.Reader, w TokenWriterI) bool {
+func (dat *DaTokenizer) TransduceTokenWriter(r io.Reader, w *TokenWriter) bool {
var a int
var t0 uint32
t := uint32(1) // Initial state