Add load factor
diff --git a/datokenizer.go b/datokenizer.go
index 4409d98..e52c8c3 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -573,6 +573,19 @@
return base
}
+func (dat *DaTokenizer) loadLevel() float64 {
+
+ nonEmpty := 0
+ all := len(dat.array) / 2
+ for x := 1; x <= len(dat.array); x = x + 2 {
+ if dat.array[x] != 0 {
+ nonEmpty++
+ }
+ }
+ fmt.Println("all:", all, "nonEmpty", nonEmpty)
+ return float64(nonEmpty) / float64(all) * 100
+}
+
// Match an input string against the double array
// FSA.
//
diff --git a/datokenizer_test.go b/datokenizer_test.go
index f9c10cc..335c7ee 100644
--- a/datokenizer_test.go
+++ b/datokenizer_test.go
@@ -1,7 +1,6 @@
package datokenizer
import (
- "fmt"
"testing"
"github.com/stretchr/testify/assert"
@@ -45,7 +44,7 @@
assert := assert.New(t)
tok := ParseFile("testdata/tokenizer.fst")
dat := tok.ToDoubleArray()
- fmt.Println("Size:", dat.maxSize)
+ assert.True(dat.loadLevel() >= 70)
assert.True(dat.Match("bau"))
assert.True(dat.Match("bad"))
assert.True(dat.Match("wald gehen"))