Test single quote handling
Change-Id: I844f14719a235670f94663ffba3ead782077c928
diff --git a/matrix_test.go b/matrix_test.go
index e7aa154..cea4acc 100644
--- a/matrix_test.go
+++ b/matrix_test.go
@@ -802,6 +802,34 @@
assert.Equal(tokens[10], ".")
assert.Equal(11, len(tokens))
+ // z.B.
+ tokens = ttokenize(mat, w, "Dies sind z.B. zwei Wörter - z. B. auch.")
+ assert.Equal(tokens[0], "Dies")
+ assert.Equal(tokens[1], "sind")
+ assert.Equal(tokens[2], "z.")
+ assert.Equal(tokens[3], "B.")
+ assert.Equal(tokens[4], "zwei")
+ assert.Equal(tokens[5], "Wörter")
+ assert.Equal(tokens[6], "-")
+ assert.Equal(tokens[7], "z.")
+ assert.Equal(tokens[8], "B.")
+ assert.Equal(tokens[9], "auch")
+ assert.Equal(tokens[10], ".")
+ assert.Equal(11, len(tokens))
+
+ // Single quote handling
+ tokens = ttokenize(mat, w, "Es heißt 'Leitungssportteams' und nicht anders.")
+ assert.Equal(tokens[0], "Es")
+ assert.Equal(tokens[1], "heißt")
+ assert.Equal(tokens[2], "'")
+ assert.Equal(tokens[3], "Leitungssportteams")
+ assert.Equal(tokens[4], "'")
+ assert.Equal(tokens[5], "und")
+ assert.Equal(tokens[6], "nicht")
+ assert.Equal(tokens[7], "anders")
+ assert.Equal(tokens[8], ".")
+ assert.Equal(9, len(tokens))
+
/*
@Test
public void englishTokenizerSeparatesEnglishContractionsAndClitics () {