Introduce EOT (end-of-transmission) marker
Change-Id: I7946e95c80fd7cd6ac1e0dd2fe5b188105f30534
diff --git a/token_writer.go b/token_writer.go
index a10f112..3ddcf27 100644
--- a/token_writer.go
+++ b/token_writer.go
@@ -6,7 +6,8 @@
)
type TokenWriterI interface {
- SentenceEnd()
+ SentenceEnd(int)
+ TextEnd(int)
Token(int, []rune)
Flush() error
}
@@ -21,10 +22,15 @@
return &TokenWriterSimple{bufio.NewWriter(w)}
}
-func (tw *TokenWriterSimple) SentenceEnd() {
+func (tw *TokenWriterSimple) SentenceEnd(_ int) {
tw.writer.WriteRune('\n')
}
+func (tw *TokenWriterSimple) TextEnd(_ int) {
+ tw.writer.WriteRune('\n')
+ tw.writer.Flush()
+}
+
func (tw *TokenWriterSimple) Token(_ int, buf []rune) {
tw.writer.WriteString(string(buf))
tw.writer.WriteRune('\n')