| Akron | 8ef408b | 2021-08-02 22:11:04 +0200 | [diff] [blame] | 1 | package datokenizer |
| 2 | |
| 3 | import ( |
| Akron | 8ef408b | 2021-08-02 22:11:04 +0200 | [diff] [blame] | 4 | "testing" |
| 5 | |
| 6 | "github.com/stretchr/testify/assert" |
| 7 | ) |
| 8 | |
| 9 | func TestSimpleString(t *testing.T) { |
| 10 | assert := assert.New(t) |
| 11 | |
| 12 | // bau | bauamt |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 13 | tok := ParseFile("testdata/bauamt.fst") |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 14 | dat := tok.ToDoubleArray() |
| 15 | assert.True(dat.Match("bau")) |
| 16 | assert.True(dat.Match("bauamt")) |
| 17 | assert.False(dat.Match("baum")) |
| Akron | 8ef408b | 2021-08-02 22:11:04 +0200 | [diff] [blame] | 18 | } |
| Akron | 75ebe7f | 2021-08-03 10:34:10 +0200 | [diff] [blame] | 19 | |
| 20 | func TestSimpleBranches(t *testing.T) { |
| 21 | assert := assert.New(t) |
| 22 | |
| 23 | // (bau | wahl) (amt | en) |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 24 | tok := ParseFile("testdata/wahlamt.fst") |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 25 | dat := tok.ToDoubleArray() |
| 26 | assert.False(dat.Match("bau")) |
| 27 | assert.True(dat.Match("bauamt")) |
| 28 | assert.True(dat.Match("wahlamt")) |
| 29 | assert.True(dat.Match("bauen")) |
| 30 | assert.True(dat.Match("wahlen")) |
| 31 | assert.False(dat.Match("baum")) |
| Akron | 75ebe7f | 2021-08-03 10:34:10 +0200 | [diff] [blame] | 32 | } |
| Akron | 730a79c | 2021-08-03 11:05:29 +0200 | [diff] [blame] | 33 | |
| 34 | func TestSimpleTokenizer(t *testing.T) { |
| 35 | assert := assert.New(t) |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 36 | tok := ParseFile("testdata/simpletok.fst") |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 37 | dat := tok.ToDoubleArray() |
| 38 | assert.True(dat.Match("bau")) |
| 39 | assert.True(dat.Match("bad")) |
| 40 | assert.True(dat.Match("wald gehen")) |
| Akron | 730a79c | 2021-08-03 11:05:29 +0200 | [diff] [blame] | 41 | } |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 42 | |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 43 | func TestFullTokenizer(t *testing.T) { |
| 44 | assert := assert.New(t) |
| 45 | tok := ParseFile("testdata/tokenizer.fst") |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 46 | dat := tok.ToDoubleArray() |
| Akron | d66a926 | 2021-08-03 17:09:09 +0200 | [diff] [blame^] | 47 | assert.True(dat.loadLevel() >= 70) |
| Akron | f2120ca | 2021-08-03 16:26:41 +0200 | [diff] [blame] | 48 | assert.True(dat.Match("bau")) |
| 49 | assert.True(dat.Match("bad")) |
| 50 | assert.True(dat.Match("wald gehen")) |
| Akron | 740f3d7 | 2021-08-03 12:12:34 +0200 | [diff] [blame] | 51 | } |