Add single character abbreviations
Change-Id: I8697f92adb3c6ad130b55260b1829bd61673a9f2
diff --git a/matrix_test.go b/matrix_test.go
index bc755b3..85dd796 100644
--- a/matrix_test.go
+++ b/matrix_test.go
@@ -729,6 +729,21 @@
assert.Equal(".", tokens[9])
assert.Equal(10, len(tokens))
+ // waste example
+ tokens = ttokenize(mat, w, "Am 24.1.1806 feierte E. T. A. Hoffmann seinen 30. Geburtstag.")
+ assert.Equal(tokens[0], "Am")
+ assert.Equal(tokens[1], "24.1.1806")
+ assert.Equal(tokens[2], "feierte")
+ assert.Equal(tokens[3], "E.")
+ assert.Equal(tokens[4], "T.")
+ assert.Equal(tokens[5], "A.")
+ assert.Equal(tokens[6], "Hoffmann")
+ assert.Equal(tokens[7], "seinen")
+ assert.Equal(tokens[8], "30.")
+ assert.Equal(tokens[9], "Geburtstag")
+ assert.Equal(tokens[10], ".")
+ assert.Equal(11, len(tokens))
+
/*
@Test
public void englishTokenizerSeparatesEnglishContractionsAndClitics () {