Support offsets in token writer
Change-Id: I1a75f8b8b3e00a0d469580cb1abee72153bbe362
diff --git a/datok.go b/datok.go
index 45a9a19..f8fccaa 100644
--- a/datok.go
+++ b/datok.go
@@ -941,7 +941,7 @@
if DEBUG {
fmt.Println("-> Flush buffer: [", string(buffer[bufft:buffc]), "]", showBuffer(buffer, buffc, buffi))
}
- w.Token(0, buffer[bufft:buffc])
+ w.Token(bufft, buffer[:buffc])
rewindBuffer = true
sentenceEnd = false
textEnd = false
diff --git a/matrix.go b/matrix.go
index 98fc32c..545cd07 100644
--- a/matrix.go
+++ b/matrix.go
@@ -493,7 +493,7 @@
if DEBUG {
fmt.Println("-> Flush buffer: [", string(buffer[bufft:buffc]), "]", showBufferNew(buffer, bufft, buffc, buffi))
}
- w.Token(0, buffer[bufft:buffc])
+ w.Token(bufft, buffer[:buffc])
rewindBuffer = true
sentenceEnd = false
textEnd = false
diff --git a/token_writer.go b/token_writer.go
index 3ddcf27..e1fabdf 100644
--- a/token_writer.go
+++ b/token_writer.go
@@ -31,8 +31,8 @@
tw.writer.Flush()
}
-func (tw *TokenWriterSimple) Token(_ int, buf []rune) {
- tw.writer.WriteString(string(buf))
+func (tw *TokenWriterSimple) Token(offset int, buf []rune) {
+ tw.writer.WriteString(string(buf[offset:]))
tw.writer.WriteRune('\n')
}
diff --git a/token_writer_test.go b/token_writer_test.go
index 84a4074..8ab6ed0 100644
--- a/token_writer_test.go
+++ b/token_writer_test.go
@@ -19,7 +19,7 @@
tws.Token(0, []rune{'a', 'b', 'c'})
- tws.Token(0, []rune{'d', 'e', 'f'})
+ tws.Token(1, []rune{'d', 'e', 'f'})
tws.SentenceEnd(0)
@@ -27,5 +27,5 @@
tws.Flush()
- assert.Equal("abc\ndef\n\n\n", w.String())
+ assert.Equal("abc\nef\n\n\n", w.String())
}