Fix buffer bug in token writer
Change-Id: I615618fe7833e1b97ae86b23d1fee760401154db
diff --git a/matrix.go b/matrix.go
index 1861528..eb88086 100644
--- a/matrix.go
+++ b/matrix.go
@@ -503,13 +503,21 @@
// token and start blank at the root node of the automaton for the remaining data.
// It may be beneficial to have something like a "drop()" event to capture these cases,
// as they are likely the result of a bad automaton design.
- if buffc-bufft == 0 {
+
+ // fmt.Println("Problem", len(buffer), buffc, bufft)
+
+ if buffc-bufft <= 0 {
buffc++
+ if buffc == 0 {
+ eof = true
+ break
+ }
}
if DEBUG {
log.Println("-> Flush buffer: [", string(buffer[bufft:buffc]), "]", showBufferNew(buffer, bufft, buffc, buffi))
}
+
w.Token(bufft, buffer[:buffc])
sentenceEnd = false