Fix sentence splitting tests
diff --git a/datokenizer.go b/datokenizer.go
index d862164..c37aaeb 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -1347,6 +1347,7 @@
fmt.Println("-> Flush buffer: [", string(data), "]")
}
writer.Write(data)
+
if dat.isTokenEnd(t) {
writer.WriteRune('\n')
if DEBUG {
@@ -1355,6 +1356,16 @@
}
}
+ // Add an additional sentence ending, if the file is over but no explicit
+ // sentence split was reached. This may be controversial and therefore
+ // optional via parameter.
+ if !dat.isTokenEnd(t0) {
+ writer.WriteRune('\n')
+ if DEBUG {
+ fmt.Println("-> Newline")
+ }
+ }
+
// There may be a new line at the end, from an epsilon, so we go on!
return true
}