Use serialized matrix representation in test suite
diff --git a/datok_test.go b/datok_test.go
index 4b4bd1e..d1c0165 100644
--- a/datok_test.go
+++ b/datok_test.go
@@ -1013,3 +1013,8 @@
// BenchmarkToDoubleArray-4 69192 17355 ns/op 10704 B/op 29 allocs/op
// BenchmarkToDoubleArrayLarger-4 16 65042885 ns/op 6357794 B/op 2576 allocs/op
// BenchmarkTransduceMatrix-4 45404 25156 ns/op 8240 B/op 3 allocs/op
+// 2021-10-02
+// BenchmarkTransduce-4 47676 25398 ns/op 8240 B/op 3 allocs/op
+// BenchmarkToDoubleArray-4 71919 16083 ns/op 10702 B/op 29 allocs/op
+// BenchmarkToDoubleArrayLarger-4 16 68012819 ns/op 6357920 B/op 2578 allocs/op
+// BenchmarkTransduceMatrix-4 51529 23678 ns/op 8240 B/op 3 allocs/op
diff --git a/matrix_test.go b/matrix_test.go
index 71a4fb5..b40fb84 100644
--- a/matrix_test.go
+++ b/matrix_test.go
@@ -125,10 +125,7 @@
func TestFullTokenizerMatrixTransduce(t *testing.T) {
assert := assert.New(t)
- foma := LoadFomaFile("testdata/tokenizer.fst")
- assert.NotNil(foma)
-
- mat := foma.ToMatrix()
+ mat := LoadMatrixFile("testdata/tokenizer.matok")
assert.NotNil(mat)
@@ -157,10 +154,7 @@
func TestFullTokenizerMatrixSentenceSplitter(t *testing.T) {
assert := assert.New(t)
- foma := LoadFomaFile("testdata/tokenizer.fst")
- assert.NotNil(foma)
-
- mat := foma.ToMatrix()
+ mat := LoadMatrixFile("testdata/tokenizer.matok")
b := make([]byte, 0, 2048)
w := bytes.NewBuffer(b)
@@ -249,10 +243,7 @@
func TestFullTokenizerMatrixTokenSplitter(t *testing.T) {
assert := assert.New(t)
- foma := LoadFomaFile("testdata/tokenizer.fst")
- assert.NotNil(foma)
-
- mat := foma.ToMatrix()
+ mat := LoadMatrixFile("testdata/tokenizer.matok")
b := make([]byte, 0, 2048)
w := bytes.NewBuffer(b)
@@ -790,10 +781,8 @@
func TestFullTokenizerMatrixXML(t *testing.T) {
assert := assert.New(t)
- foma := LoadFomaFile("testdata/tokenizer.fst")
- assert.NotNil(foma)
+ mat := LoadMatrixFile("testdata/tokenizer.matok")
- mat := foma.ToMatrix()
assert.NotNil(mat)
b := make([]byte, 0, 2048)
@@ -845,8 +834,7 @@
Mach's macht's was'n ist's haste willste kannste biste kriegste.`
r := strings.NewReader(s)
- foma := LoadFomaFile("testdata/tokenizer.fst")
- mat := foma.ToMatrix()
+ mat := LoadMatrixFile("testdata/tokenizer.matok")
b.ResetTimer()
diff --git a/testdata/tokenizer.matok b/testdata/tokenizer.matok
new file mode 100644
index 0000000..9676beb
--- /dev/null
+++ b/testdata/tokenizer.matok
Binary files differ