| Akron | 8ef408b | 2021-08-02 22:11:04 +0200 | [diff] [blame] | 1 | package datokenizer |
| 2 | |
| 3 | import ( |
| Akron | 6247a5d | 2021-08-03 19:18:28 +0200 | [diff] [blame] | 4 | "bytes" |
| Akron | 8ef408b | 2021-08-02 22:11:04 +0200 | [diff] [blame] | 5 | "testing" |
| 6 | |
| 7 | "github.com/stretchr/testify/assert" |
| 8 | ) |
| 9 | |
| 10 | func TestSimpleString(t *testing.T) { |
| 11 | assert := assert.New(t) |
| 12 | |
| 13 | // bau | bauamt |
| Akron | 64ffd9a | 2021-08-03 19:55:21 +0200 | [diff] [blame] | 14 | tok := LoadFomaFile("testdata/bauamt.fst") |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 15 | dat := tok.ToDoubleArray() |
| 16 | assert.True(dat.Match("bau")) |
| 17 | assert.True(dat.Match("bauamt")) |
| 18 | assert.False(dat.Match("baum")) |
| Akron | 8ef408b | 2021-08-02 22:11:04 +0200 | [diff] [blame] | 19 | } |
| Akron | 75ebe7f | 2021-08-03 10:34:10 +0200 | [diff] [blame] | 20 | |
| 21 | func TestSimpleBranches(t *testing.T) { |
| 22 | assert := assert.New(t) |
| 23 | |
| 24 | // (bau | wahl) (amt | en) |
| Akron | 64ffd9a | 2021-08-03 19:55:21 +0200 | [diff] [blame] | 25 | tok := LoadFomaFile("testdata/wahlamt.fst") |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 26 | dat := tok.ToDoubleArray() |
| 27 | assert.False(dat.Match("bau")) |
| 28 | assert.True(dat.Match("bauamt")) |
| 29 | assert.True(dat.Match("wahlamt")) |
| 30 | assert.True(dat.Match("bauen")) |
| 31 | assert.True(dat.Match("wahlen")) |
| 32 | assert.False(dat.Match("baum")) |
| Akron | 75ebe7f | 2021-08-03 10:34:10 +0200 | [diff] [blame] | 33 | } |
| Akron | 730a79c | 2021-08-03 11:05:29 +0200 | [diff] [blame] | 34 | |
| 35 | func TestSimpleTokenizer(t *testing.T) { |
| 36 | assert := assert.New(t) |
| Akron | 64ffd9a | 2021-08-03 19:55:21 +0200 | [diff] [blame] | 37 | tok := LoadFomaFile("testdata/simpletok.fst") |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 38 | dat := tok.ToDoubleArray() |
| 39 | assert.True(dat.Match("bau")) |
| 40 | assert.True(dat.Match("bad")) |
| 41 | assert.True(dat.Match("wald gehen")) |
| Akron | 730a79c | 2021-08-03 11:05:29 +0200 | [diff] [blame] | 42 | } |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 43 | |
| Akron | 068874c | 2021-08-04 15:19:56 +0200 | [diff] [blame] | 44 | func TestSimpleTokenizerTransduce(t *testing.T) { |
| Akron | 84d68e6 | 2021-08-04 17:06:52 +0200 | [diff] [blame^] | 45 | assert := assert.New(t) |
| 46 | tok := LoadFomaFile("testdata/simpletok.fst") |
| 47 | // tok := LoadFomaFile("testdata/tokenizer.fst") |
| 48 | dat := tok.ToDoubleArray() |
| 49 | // assert.True(dat.Transduce("bau")) |
| 50 | // assert.True(dat.Match("bad")) |
| 51 | assert.True(dat.Transduce("wald gehen")) |
| 52 | // assert.True(dat.Transduce("wald gehen")) |
| 53 | assert.Fail("!") |
| Akron | 068874c | 2021-08-04 15:19:56 +0200 | [diff] [blame] | 54 | } |
| 55 | |
| Akron | 6247a5d | 2021-08-03 19:18:28 +0200 | [diff] [blame] | 56 | func TestWriteTokenizer(t *testing.T) { |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 57 | assert := assert.New(t) |
| Akron | 64ffd9a | 2021-08-03 19:55:21 +0200 | [diff] [blame] | 58 | tok := LoadFomaFile("testdata/simpletok.fst") |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 59 | dat := tok.ToDoubleArray() |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 60 | assert.True(dat.Match("bau")) |
| 61 | assert.True(dat.Match("bad")) |
| 62 | assert.True(dat.Match("wald gehen")) |
| Akron | 6247a5d | 2021-08-03 19:18:28 +0200 | [diff] [blame] | 63 | |
| Akron | 03a3c61 | 2021-08-04 11:51:27 +0200 | [diff] [blame] | 64 | assert.True(dat.LoadFactor() >= 70) |
| Akron | 6247a5d | 2021-08-03 19:18:28 +0200 | [diff] [blame] | 65 | |
| 66 | b := make([]byte, 1024) |
| 67 | buf := bytes.NewBuffer(b) |
| 68 | n, err := dat.WriteTo(buf) |
| 69 | assert.Nil(err) |
| 70 | assert.Equal(n, int64(186)) |
| 71 | } |
| 72 | |
| 73 | func TestFullTokenizer(t *testing.T) { |
| Akron | 2a4b929 | 2021-08-04 15:35:22 +0200 | [diff] [blame] | 74 | /* |
| 75 | assert := assert.New(t) |
| 76 | tok := LoadFomaFile("testdata/tokenizer.fst") |
| 77 | dat := tok.ToDoubleArray() |
| 78 | assert.True(dat.LoadFactor() >= 70) |
| 79 | assert.True(dat.Match("bau")) |
| 80 | assert.True(dat.Match("bad")) |
| 81 | assert.True(dat.Match("wald gehen")) |
| 82 | */ |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 83 | } |