Introduced multiple gender forms for nouns in german
Change-Id: Ic98042ccc01903ec279f9d58a4c3a11504dc4025
diff --git a/src/de/tokenizer.xfst b/src/de/tokenizer.xfst
index aa2092b..a6bff24 100644
--- a/src/de/tokenizer.xfst
+++ b/src/de/tokenizer.xfst
@@ -46,10 +46,17 @@
define NotSmallCaps [? - a - b - c - d - e - f - g - h - i - j - k - l - m - n - o - p - q - r - s - t - u - v - w - x - y - z - ü - ö - ä - è - é - ú - á - â - ê - î - ô - û];
-define Word Char+ ([Apos|Asterisk] Char+)* ([s|S] [%’|%`]);
+define BaseWord Char+ ([Apos|Asterisk] Char+)* ([s|S] [%’|%`]);
define Plusampersand @txt"de/plusampersand.txt";
-define Word [Plusampersand | Word] (Dash [Plusampersand | Word])*;
+
+source de/gender.xfst
+
+define CoreWord [Plusampersand | BaseWord] (Dash [Plusampersand | BaseWord])*;
+define Word [ CoreWord | CoreWord GenderEndings ];
+
+! [ Word GenderEndings || WS] @-> ... NLout,
+
! Abbreviations and Initials
! The abbreviation list is part of the sentence splitter tool