Remove unnecessary allocation for buffer recasting
diff --git a/.gitignore b/.gitignore
index e4b40a2..a3d640f 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,6 @@
sandbox
+todo.org
+tmp
tokenizer.code-workspace
~*
.*
diff --git a/datokenizer.go b/datokenizer.go
index 3c6310b..f2d9304 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -324,7 +324,7 @@
// Only a limited list of transitions are allowed
if inSym != outSym {
- if outSym == tok.tokenend {
+ if outSym == tok.tokenend && inSym == tok.epsilon {
tokenend = true
} else if outSym == tok.epsilon {
nontoken = true
@@ -1180,11 +1180,10 @@
if dat.isTokenEnd(t) {
if buffi > 0 {
- data := []byte(string(buffer[:buffo]))
if DEBUG {
- fmt.Println("-> Flush buffer: [", string(data), "]", showBuffer(buffer, buffo, buffi))
+ fmt.Println("-> Flush buffer: [", string(buffer[:buffo]), "]", showBuffer(buffer, buffo, buffi))
}
- writer.Write(data)
+ writer.WriteString(string(buffer[:buffo]))
rewindBuffer = true
}
if DEBUG {
@@ -1240,11 +1239,10 @@
if dat.getCheck(dat.getBase(t)+uint32(dat.final)) == t {
if buffi > 0 {
- data := []byte(string(buffer[:buffi]))
if DEBUG {
- fmt.Println("-> Flush buffer: [", string(data), "]")
+ fmt.Println("-> Flush buffer: [", string(buffer[:buffi]), "]")
}
- writer.Write(data)
+ writer.WriteString(string(buffer[:buffi]))
if dat.isTokenEnd(t) {
writer.WriteRune('\n')
@@ -1271,20 +1269,18 @@
// Check epsilon transitions until a final state is reached
t0 = t
t = dat.getBase(t0) + uint32(dat.epsilon)
+ a = dat.epsilon
+ newchar = false
if dat.getCheck(t) == t0 {
// Remember state for backtracking to last tokenend state
- a = dat.epsilon
- newchar = false
goto PARSECHAR
} else if epsilonState != 0 {
t0 = epsilonState
epsilonState = 0 // reset
buffo = epsilonOffset
- a = dat.epsilon
if DEBUG {
fmt.Println("Get from epsilon stack and set buffo!", showBuffer(buffer, buffo, buffi))
}
- newchar = false
goto PARSECHAR
}
return false
diff --git a/datokenizer_test.go b/datokenizer_test.go
index 5faf0fc..e7e37d5 100644
--- a/datokenizer_test.go
+++ b/datokenizer_test.go
@@ -847,9 +847,9 @@
os.Exit(1)
}
}
- // 2021-08-11
- // BenchmarkTransduce-4 18669 68511 ns/op
- // BenchmarkTransduce-4 18656 63068 ns/op
- // BenchmarkTransduce-4 15210 67403 ns/op
- // BenchmarkTransduce-4 18358 62233 ns/op
+ // 2021-08-11 (go 1.16)
+ // go test -bench=. -test.benchmem
+ // BenchmarkTransduce-4 19069 60609 ns/op 11048 B/op 137 allocs/op
+ // 2021-08-112 (go 1.16)
+ // BenchmarkTransduce-4 20833 55241 ns/op 9676 B/op 3 allocs/op
}