Fix parsing of end states
diff --git a/datokenizer.go b/datokenizer.go
index f455ec3..96243f1 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -295,6 +295,20 @@
if elemint[1] == -1 {
state = elemint[0]
final = elemint[3]
+
+ // Final state that has no outgoing edges
+ if final == 1 {
+
+ // Initialize outgoing states
+ if tok.transitions[state+1] == nil {
+ tok.transitions[state+1] = make(map[int]*edge)
+ }
+
+ // TODO:
+ // Maybe this is less relevant for tokenizers
+ tok.transitions[state+1][tok.final] = &edge{}
+ }
+ continue
} else {
state = elemint[0]
inSym = elemint[1]