Fix sentence splitting
diff --git a/datokenizer.go b/datokenizer.go
index 7aea95a..d862164 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -1176,6 +1176,9 @@
if newchar {
// Get from reader if buffer is empty
if buffo >= buffi {
+ if eof {
+ break
+ }
char, _, err = reader.ReadRune()
// No more runes to read
@@ -1282,12 +1285,14 @@
data := []byte(string(buffer[:buffo]))
if DEBUG {
fmt.Println("-> Flush buffer: [", string(data), "]", showBuffer(buffer, buffo, buffi))
- fmt.Println("-> Newline")
}
writer.Write(data)
- writer.WriteRune('\n')
rewindBuffer = true
}
+ if DEBUG {
+ fmt.Println("-> Newline")
+ }
+ writer.WriteRune('\n')
}
// Rewind the buffer if necessary
@@ -1315,10 +1320,6 @@
}
}
- if eof {
- break
- }
-
newchar = true
// TODO: