commit 9fb63afd11678fead154ebeefcc182a541b24e3e
author Akron <nils@diewald-online.de> Thu Oct 28 01:15:53 2021 +0200
committer Akron <nils@diewald-online.de> Thu Oct 28 01:15:53 2021 +0200
tree 9d0129b21bf37e962822cb002a1bec70dae5e7d2
parent 7035d2e4dab6e2e3d697f59de17e056e045e25c6

Optimize tests by avoiding reload of tokenizers

Change-Id: Ia70dab59fc3cfe5e47a1540724336214addb824e

matrix_test.go

diff --git a/matrix_test.go b/matrix_test.go
index e27dd12..1a3f7b0 100644
--- a/matrix_test.go
+++ b/matrix_test.go
@@ -20,6 +20,8 @@
 Archive:  Ich bin kein zip. D'dorf Ku'damm Lu'hafen M'gladbach W'schaft.
 Mach's macht's was'n ist's haste willste kannste biste kriegste.`
 
+var mat *MatrixTokenizer
+
 func TestMatrixFullTokenizer(t *testing.T) {
 	assert := assert.New(t)
 	foma := LoadFomaFile("testdata/simpletok.fst")
@@ -162,7 +164,9 @@
 func TestMatrixFullTokenizerTransduce(t *testing.T) {
 	assert := assert.New(t)
 
-	mat := LoadMatrixFile("testdata/tokenizer.matok")
+	if mat == nil {
+		mat = LoadMatrixFile("testdata/tokenizer.matok")
+	}
 
 	assert.NotNil(mat)
 
@@ -191,7 +195,10 @@
 
 func TestMatrixFullTokenizerMatrixSentenceSplitter(t *testing.T) {
 	assert := assert.New(t)
-	mat := LoadMatrixFile("testdata/tokenizer.matok")
+
+	if mat == nil {
+		mat = LoadMatrixFile("testdata/tokenizer.matok")
+	}
 
 	b := make([]byte, 0, 2048)
 	w := bytes.NewBuffer(b)
@@ -288,7 +295,9 @@
 func TestMatrixFullTokenizerTokenSplitter(t *testing.T) {
 	assert := assert.New(t)
 
-	mat := LoadMatrixFile("testdata/tokenizer.matok")
+	if mat == nil {
+		mat = LoadMatrixFile("testdata/tokenizer.matok")
+	}
 
 	b := make([]byte, 0, 2048)
 	w := bytes.NewBuffer(b)
@@ -826,7 +835,9 @@
 func TestMatrixFullTokenizerXML(t *testing.T) {
 	assert := assert.New(t)
 
-	mat := LoadMatrixFile("testdata/tokenizer.matok")
+	if mat == nil {
+		mat = LoadMatrixFile("testdata/tokenizer.matok")
+	}
 
 	assert.NotNil(mat)
 
@@ -867,7 +878,9 @@
 func TestMatokDatokEquivalence(t *testing.T) {
 	assert := assert.New(t)
 
-	mat := LoadMatrixFile("testdata/tokenizer.matok")
+	if mat == nil {
+		mat = LoadMatrixFile("testdata/tokenizer.matok")
+	}
 	dat := LoadDatokFile("testdata/tokenizer.datok")
 
 	r := strings.NewReader(s)
@@ -894,7 +907,9 @@
 func TestMatrixFullTokenizerCallbackTransduce(t *testing.T) {
 	assert := assert.New(t)
 
-	mat := LoadMatrixFile("testdata/tokenizer.matok")
+	if mat == nil {
+		mat = LoadMatrixFile("testdata/tokenizer.matok")
+	}
 
 	assert.NotNil(mat)
 
@@ -911,7 +926,9 @@
 func TestMatrixFullTokenizerTextTreatment(t *testing.T) {
 	assert := assert.New(t)
 
-	mat := LoadMatrixFile("testdata/tokenizer.matok")
+	if mat == nil {
+		mat = LoadMatrixFile("testdata/tokenizer.matok")
+	}
 
 	assert.NotNil(mat)
 
@@ -926,7 +943,9 @@
 func TestMatrixTrimming(t *testing.T) {
 	assert := assert.New(t)
 
-	mat := LoadMatrixFile("testdata/tokenizer.matok")
+	if mat == nil {
+		mat = LoadMatrixFile("testdata/tokenizer.matok")
+	}
 
 	assert.NotNil(mat)