Support Plusampersand words in compounds
Change-Id: I095681ece9c7e2e80fb2975eb6bf87463b17db7c
diff --git a/matrix_test.go b/matrix_test.go
index b0d2698..7200608 100644
--- a/matrix_test.go
+++ b/matrix_test.go
@@ -769,7 +769,7 @@
assert.Equal(tokens[11], ".")
assert.Equal(12, len(tokens))
- // Plusampersand compounds
+ // Plusampersand compounds (1)
tokens = ttokenize(mat, w, "Die 2G+-Regel soll weitere Covid-19-Erkrankungen reduzieren.")
assert.Equal(tokens[0], "Die")
assert.Equal(tokens[1], "2G+-Regel")
@@ -780,6 +780,13 @@
assert.Equal(tokens[6], ".")
assert.Equal(7, len(tokens))
+ // Plusampersand compounds (2)
+ tokens = ttokenize(mat, w, "Der Neu-C++-Programmierer.")
+ assert.Equal(tokens[0], "Der")
+ assert.Equal(tokens[1], "Neu-C++-Programmierer")
+ assert.Equal(tokens[2], ".")
+ assert.Equal(3, len(tokens))
+
/*
@Test
public void englishTokenizerSeparatesEnglishContractionsAndClitics () {