Correctly handle final data
diff --git a/datokenizer.go b/datokenizer.go
index b04b487..b89bccd 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -1327,10 +1327,9 @@
fmt.Println("-> Flush buffer:", string(data))
}
writer.Write(data)
- }
-
- if dat.isTokenEnd(t) {
- writer.WriteRune('\n')
+ if dat.isTokenEnd(t) {
+ writer.WriteRune('\n')
+ }
}
// There may be a new line at the end, from an epsilon, so we go on!
diff --git a/datokenizer_test.go b/datokenizer_test.go
index cfbdf05..39aa775 100644
--- a/datokenizer_test.go
+++ b/datokenizer_test.go
@@ -81,8 +81,7 @@
assert.Equal("--", tokens[2])
assert.Equal("D", tokens[3])
assert.Equal("", tokens[4])
- assert.Equal("", tokens[5])
- assert.Equal(6, len(tokens))
+ assert.Equal(5, len(tokens))
}
func TestReadWriteTokenizer(t *testing.T) {
@@ -169,6 +168,5 @@
assert.Equal("?", tokens[5])
assert.Equal("", tokens[6])
assert.Equal("", tokens[7])
- assert.Equal("", tokens[8])
- assert.Equal(9, len(tokens))
+ assert.Equal(8, len(tokens))
}