Turn states into uint32 pairs
diff --git a/datokenizer_test.go b/datokenizer_test.go
index 7f2b22f..9d96982 100644
--- a/datokenizer_test.go
+++ b/datokenizer_test.go
@@ -59,13 +59,11 @@
}
func TestFullTokenizer(t *testing.T) {
- /*
- assert := assert.New(t)
- tok := ParseFile("testdata/tokenizer.fst")
- dat := tok.ToDoubleArray()
- assert.True(dat.LoadLevel() >= 70)
- assert.True(dat.Match("bau"))
- assert.True(dat.Match("bad"))
- assert.True(dat.Match("wald gehen"))
- */
+ assert := assert.New(t)
+ tok := LoadFomaFile("testdata/tokenizer.fst")
+ dat := tok.ToDoubleArray()
+ assert.True(dat.LoadLevel() >= 70)
+ assert.True(dat.Match("bau"))
+ assert.True(dat.Match("bad"))
+ assert.True(dat.Match("wald gehen"))
}