Introduce token_writer object
This change also removes final state
sensibility from the tokenizer. Tokens
now require a tokenend transition to
be treated as complete.
diff --git a/token_writer.go b/token_writer.go
new file mode 100644
index 0000000..a10f112
--- /dev/null
+++ b/token_writer.go
@@ -0,0 +1,35 @@
+package datok
+
+import (
+ "bufio"
+ "io"
+)
+
+type TokenWriterI interface {
+ SentenceEnd()
+ Token(int, []rune)
+ Flush() error
+}
+
+var _ TokenWriterI = &TokenWriterSimple{}
+
+type TokenWriterSimple struct {
+ writer *bufio.Writer
+}
+
+func NewTokenWriterSimple(w io.Writer) *TokenWriterSimple {
+ return &TokenWriterSimple{bufio.NewWriter(w)}
+}
+
+func (tw *TokenWriterSimple) SentenceEnd() {
+ tw.writer.WriteRune('\n')
+}
+
+func (tw *TokenWriterSimple) Token(_ int, buf []rune) {
+ tw.writer.WriteString(string(buf))
+ tw.writer.WriteRune('\n')
+}
+
+func (tw *TokenWriterSimple) Flush() error {
+ return tw.writer.Flush()
+}