Support unknown and identity symbols
diff --git a/datokenizer_test.go b/datokenizer_test.go
index f714e10..64631c4 100644
--- a/datokenizer_test.go
+++ b/datokenizer_test.go
@@ -1,6 +1,7 @@
package datokenizer
import (
+ "fmt"
"testing"
"github.com/stretchr/testify/assert"
@@ -30,3 +31,17 @@
assert.True(tok.match("wahlen"))
assert.False(tok.match("baum"))
}
+
+func TestSimpleTokenizer(t *testing.T) {
+ assert := assert.New(t)
+
+ fmt.Println("-------------------")
+
+ tok := parse_file("testdata/simpletok.fst")
+ tok.buildDA()
+ assert.True(tok.match("bau"))
+ /*
+ assert.True(tok.match("bad"))
+ assert.True(tok.match("wald gehen"))
+ */
+}