Support both matrix and da in the command
diff --git a/datok_test.go b/datok_test.go
index 8c2b894..1ebb167 100644
--- a/datok_test.go
+++ b/datok_test.go
@@ -179,6 +179,17 @@
assert.True(tmatch(dat, "wald gehen"))
}
+func TestTokenizerBranch(t *testing.T) {
+ assert := assert.New(t)
+ tok := LoadTokenizerFile("testdata/simpletok.datok")
+ assert.NotNil(tok)
+ assert.Equal(tok.Type(), "DATOK")
+
+ tok = LoadTokenizerFile("testdata/simpletok.matok")
+ assert.NotNil(tok)
+ assert.Equal(tok.Type(), "MATOK")
+}
+
func XTestFullTokenizerBuild(t *testing.T) {
assert := assert.New(t)
tok := LoadFomaFile("testdata/tokenizer.fst")