Sort alphabet prior to xCheck
diff --git a/datokenizer_test.go b/datokenizer_test.go
index 4ba42b3..8cabb67 100644
--- a/datokenizer_test.go
+++ b/datokenizer_test.go
@@ -1,6 +1,7 @@
package datokenizer
import (
+ "fmt"
"testing"
"github.com/stretchr/testify/assert"
@@ -40,13 +41,12 @@
assert.True(tok.Match("wald gehen"))
}
-/*
func TestFullTokenizer(t *testing.T) {
assert := assert.New(t)
tok := ParseFile("testdata/tokenizer.fst")
tok.ToDoubleArray()
+ fmt.Println("Size:", tok.maxSize)
assert.True(tok.Match("bau"))
assert.True(tok.Match("bad"))
assert.True(tok.Match("wald gehen"))
}
-*/