Add support for epsilon symbols
diff --git a/datokenizer_test.go b/datokenizer_test.go
index 64631c4..7a20709 100644
--- a/datokenizer_test.go
+++ b/datokenizer_test.go
@@ -1,7 +1,6 @@
package datokenizer
import (
- "fmt"
"testing"
"github.com/stretchr/testify/assert"
@@ -34,14 +33,9 @@
func TestSimpleTokenizer(t *testing.T) {
assert := assert.New(t)
-
- fmt.Println("-------------------")
-
tok := parse_file("testdata/simpletok.fst")
tok.buildDA()
assert.True(tok.match("bau"))
- /*
- assert.True(tok.match("bad"))
- assert.True(tok.match("wald gehen"))
- */
+ assert.True(tok.match("bad"))
+ // assert.True(tok.match("wald gehen"))
}