Support German gender-sensitive noun endings with punctuation and braces
Resolves #132
Change-Id: I249ba02ededab065d9b7aff8393d92ad42c36df6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 93e3e77..4d63d5c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,15 @@
# Changelog
+## 2.4.0 [unreleased]
+
+* Added German gender-sensitive form tokenization:
+ - Colon forms: `Nutzer:in`, `Nutzer:innen`, `Kosovo-Albaner:innen`
+ - Slash forms: `Nutzer/in`, `Nutzer/innen`, `Nutzer/-in`, `Kosovo-Albaner/innen`
+ - Parenthetical forms: `Nutzer(in)`, `Nutzer(innen)`, `Nutzer(-in)`
+ - Kaufmann/frau pattern: `Kaufmann/frau`, `Kaufmann/-frau`, `Geschäftsmann/frau`
+ (only applies when word ends in "mann" with non-empty prefix)
+
## 2.3.1 [2026-01-28]
* Fixed soft hyphens (U+00AD) being incorrectly treated as token boundaries (issue #131)
diff --git a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/DerekoDfaTokenizer.jflex b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/DerekoDfaTokenizer.jflex
index fdd631a..8609827 100644
--- a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/DerekoDfaTokenizer.jflex
+++ b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/DerekoDfaTokenizer.jflex
@@ -649,7 +649,28 @@
\[\[+ { return currentToken();}
\]\]+ { return currentToken();}
+// Gender-sensitive forms (German-specific, via GENDER_ENDING macro in language-specific_de.jflex-macro)
+// Colon forms: Nutzer:in, Nutzer:innen, Kosovo-Albaner:innen
+({WORD}({DASH}{WORD})*):{GENDER_ENDING} { return currentToken(); }
+
+// Slash forms for -in/-innen: Nutzer/in, Nutzer/innen, Nutzer/-in, Kosovo-Albaner/innen
+({WORD}({DASH}{WORD})*){SLASH}-?{GENDER_ENDING} { return currentToken(); }
+
+// Slash forms for -frau: Kaufmann/frau, Kaufmann/-frau, Geschäftsmann/frau
+// Only applies when word ends in "mann" (with non-empty prefix before it)
+({WORD}({DASH}{WORD})*{DASH})?{MANN_WORD}{SLASH}-?{GENDER_ENDING_FRAU} { return currentToken(); }
+
+// Parenthetical forms for -in/-innen: Nutzer(in), Nutzer(innen), Nutzer(-in)
+({WORD}({DASH}{WORD})*)"("-?{GENDER_ENDING}")" { return currentToken(); }
+
+// Parenthetical forms for -frau: Kaufmann(frau), Kaufmann(-frau)
+// Only applies when word ends in "mann" (with non-empty prefix before it)
+({WORD}({DASH}{WORD})*{DASH})?{MANN_WORD}"("-?{GENDER_ENDING_FRAU}")" { return currentToken(); }
+
+
// normal stuff
+
+
// dashed words
{WORD}({DASH}{NEWLINE}*({WORD}|{OMISSIONWORD}))+ { return currentToken();}
{WORD}{DASH} { return currentToken();}
diff --git a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_de.jflex-macro b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_de.jflex-macro
index bbce886..421c637 100644
--- a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_de.jflex-macro
+++ b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_de.jflex-macro
@@ -17,3 +17,18 @@
SEABBRHYPH = (Art\.-Nr|At\.-Gew|Ba\.-Wü|Best\.-Nr|Br\.-M|Br\.-Mstr|Dipl\.-Bibl|Dipl\.-Ing|Dipl\.-Kff|Dipl\.-Kffr|Dipl\.-Kfm|Dipl\.-Kfr|Dipl\.-Psych|Do\.-Gge|Ers\.-D|Erschl\.-Geb|Erw\.-Bldg|Geb\.-T|H\.-I|H\.-Qu|Hd\.-Bibl|Hs\.-Nr|klass\.-lat|Krim\.-Ob\.-Insp|Kto\.-Nr|L\.-Abg|M\.-Schr|Orch\.-Bes|Priv\.-Doz|prov\.-fr|Proz\.-Bev|r\.-k|Reg\.-Bez|Rg\.-Präs|röm\.-kath|S\.-Wk|St\.-Nr)
//#endif
+// Gender-sensitive endings (German)
+// Matches patterns like: in, innen, In, Innen, IN, INNEN (case-insensitive)
+GENDER_ENDING_IN = ([iI][nN]|[iI][nN][nN][eE][nN])
+
+// Gender-sensitive endings with frau/frauen (lowercase only - capitalized Frau is a standalone word)
+// Note: This is now only used with MANN_WORD, not in general GENDER_ENDING
+GENDER_ENDING_FRAU = (frau(en)?)
+
+// General gender endings (only -in/-innen forms for colon, slash, parenthetical)
+GENDER_ENDING = ({GENDER_ENDING_IN})
+
+// Words ending in "mann" (with non-empty prefix) for Kaufmann/frau pattern
+// Matches: Kaufmann, Geschäftsmann, etc. but NOT just "mann"
+MANN_WORD = ({LETTER}+[Mm][Aa][Nn][Nn])
+
diff --git a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_en.jflex-macro b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_en.jflex-macro
index ffe6409..411c7ad 100644
--- a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_en.jflex-macro
+++ b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_en.jflex-macro
@@ -6,3 +6,9 @@
SEABBRHYPH = ("DUMMY")
//#endif
CLITIC = ({ENGLISH_CLITIC}|{FRENCH_CLITIC})
+
+// Gender-sensitive endings not used for English (German-only feature)
+// Use NUL character so the rules never match
+GENDER_ENDING = (\u0000)
+GENDER_ENDING_FRAU = (\u0000)
+MANN_WORD = (\u0000)
diff --git a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_fr.jflex-macro b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_fr.jflex-macro
index 5a6f88b..4cb9166 100644
--- a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_fr.jflex-macro
+++ b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/language-specific_fr.jflex-macro
@@ -8,3 +8,9 @@
SEABBRHYPH = ("DUMMY")
//#endif
CLITIC = ({ENGLISH_CLITIC}|{FRENCH_CLITIC})
+
+// Gender-sensitive endings not used for French (German-only feature)
+// Use NUL character so the rules never match
+GENDER_ENDING = (\u0000)
+GENDER_ENDING_FRAU = (\u0000)
+MANN_WORD = (\u0000)
diff --git a/src/test/java/de/ids_mannheim/korap/tokenizer/TokenizerTest.java b/src/test/java/de/ids_mannheim/korap/tokenizer/TokenizerTest.java
index 82f758b..1d8961a 100644
--- a/src/test/java/de/ids_mannheim/korap/tokenizer/TokenizerTest.java
+++ b/src/test/java/de/ids_mannheim/korap/tokenizer/TokenizerTest.java
@@ -825,4 +825,235 @@
assertEquals("Donau\u00ADdampf\u00ADschiff", tokens[0]);
assertEquals(1, tokens.length);
}
+
+ // Regression tests for German gender-sensitive forms
+ @Test
+ public void testGenderSensitiveColonForms() {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+
+ // Basic colon forms with -in/-innen
+ String[] tokens = tok.tokenize("Die Schüler:innen und Lehrer:in kamen.");
+ assertEquals("Die", tokens[0]);
+ assertEquals("Schüler:innen", tokens[1]);
+ assertEquals("und", tokens[2]);
+ assertEquals("Lehrer:in", tokens[3]);
+ assertEquals("kamen", tokens[4]);
+ assertEquals(".", tokens[5]);
+ assertEquals(6, tokens.length);
+
+ // More colon examples
+ tokens = tok.tokenize("Künstler:innen Mitarbeiter:innen Bürger:innen");
+ assertEquals("Künstler:innen", tokens[0]);
+ assertEquals("Mitarbeiter:innen", tokens[1]);
+ assertEquals("Bürger:innen", tokens[2]);
+ assertEquals(3, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveSlashForms() {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+
+ // Basic slash forms
+ String[] tokens = tok.tokenize("Autor/in Autor/innen Teilnehmer/innen");
+ assertEquals("Autor/in", tokens[0]);
+ assertEquals("Autor/innen", tokens[1]);
+ assertEquals("Teilnehmer/innen", tokens[2]);
+ assertEquals(3, tokens.length);
+
+ // Slash forms with hyphen: /-in, /-innen, /-frau
+ tokens = tok.tokenize("Kaufmann/-frau und Fachmann/-frau");
+ assertEquals("Kaufmann/-frau", tokens[0]);
+ assertEquals("und", tokens[1]);
+ assertEquals("Fachmann/-frau", tokens[2]);
+ assertEquals(3, tokens.length);
+
+ // Slash forms without hyphen for frau (lowercase only)
+ tokens = tok.tokenize("Kaufmann/frau ist auch korrekt.");
+ assertEquals("Kaufmann/frau", tokens[0]);
+ assertEquals("ist", tokens[1]);
+ assertEquals("auch", tokens[2]);
+ assertEquals("korrekt", tokens[3]);
+ assertEquals(".", tokens[4]);
+ assertEquals(5, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveParentheticalForms() {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+
+ // Basic parenthetical forms
+ String[] tokens = tok.tokenize("Schüler(innen) und Lehrer(in) kamen.");
+ assertEquals("Schüler(innen)", tokens[0]);
+ assertEquals("und", tokens[1]);
+ assertEquals("Lehrer(in)", tokens[2]);
+ assertEquals("kamen", tokens[3]);
+ assertEquals(".", tokens[4]);
+ assertEquals(5, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveCompoundWords() {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+
+ // Compound words with hyphen + gender ending
+ String[] tokens = tok.tokenize("Die Kosovo-Albaner/innen und Kosovo-Albaner:innen trafen sich.");
+ assertEquals("Die", tokens[0]);
+ assertEquals("Kosovo-Albaner/innen", tokens[1]);
+ assertEquals("und", tokens[2]);
+ assertEquals("Kosovo-Albaner:innen", tokens[3]);
+ assertEquals("trafen", tokens[4]);
+ assertEquals("sich", tokens[5]);
+ assertEquals(".", tokens[6]);
+ assertEquals(7, tokens.length);
+
+ // With hyphen: Kosovo-Albaner/-innen
+ tokens = tok.tokenize("Kosovo-Albaner/-innen kamen.");
+ assertEquals("Kosovo-Albaner/-innen", tokens[0]);
+ assertEquals("kamen", tokens[1]);
+ assertEquals(".", tokens[2]);
+ assertEquals(3, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveShouldSeparateMannFrau() {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+
+ // Mann/Frau should be separated (capital F = standalone word, not suffix)
+ String[] tokens = tok.tokenize("Ob Mann/Frau das will?");
+ assertEquals("Ob", tokens[0]);
+ assertEquals("Mann", tokens[1]);
+ assertEquals("/", tokens[2]);
+ assertEquals("Frau", tokens[3]);
+ assertEquals("das", tokens[4]);
+ assertEquals("will", tokens[5]);
+ assertEquals("?", tokens[6]);
+ assertEquals(7, tokens.length);
+
+ // Also Männer/Frauen
+ tokens = tok.tokenize("Männer/Frauen sind willkommen.");
+ assertEquals("Männer", tokens[0]);
+ assertEquals("/", tokens[1]);
+ assertEquals("Frauen", tokens[2]);
+ assertEquals("sind", tokens[3]);
+ assertEquals("willkommen", tokens[4]);
+ assertEquals(".", tokens[5]);
+ assertEquals(6, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveSlashFrauOnlyAfterMann() {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+
+ // /frau should only be joined when word ends in "mann"
+ // "xxx/frau" where xxx doesn't end in "mann" should be SEPARATED
+ String[] tokens = tok.tokenize("xxx/frau sollte getrennt sein.");
+ assertEquals("xxx", tokens[0]);
+ assertEquals("/", tokens[1]);
+ assertEquals("frau", tokens[2]);
+ assertEquals("sollte", tokens[3]);
+ assertEquals("getrennt", tokens[4]);
+ assertEquals("sein", tokens[5]);
+ assertEquals(".", tokens[6]);
+ assertEquals(7, tokens.length);
+
+ // But Kaufmann/frau should be one token (word ends in "mann")
+ tokens = tok.tokenize("Kaufmann/frau ist ein Beruf.");
+ assertEquals("Kaufmann/frau", tokens[0]);
+ assertEquals("ist", tokens[1]);
+ assertEquals("ein", tokens[2]);
+ assertEquals("Beruf", tokens[3]);
+ assertEquals(".", tokens[4]);
+ assertEquals(5, tokens.length);
+
+ // And Fachmann/-frau should be one token
+ tokens = tok.tokenize("Fachmann/-frau gesucht");
+ assertEquals("Fachmann/-frau", tokens[0]);
+ assertEquals("gesucht", tokens[1]);
+ assertEquals(2, tokens.length);
+
+ // Geschäftsmann/frau should also be one token
+ tokens = tok.tokenize("Ein Geschäftsmann/frau wird gesucht.");
+ assertEquals("Ein", tokens[0]);
+ assertEquals("Geschäftsmann/frau", tokens[1]);
+ assertEquals("wird", tokens[2]);
+ assertEquals("gesucht", tokens[3]);
+ assertEquals(".", tokens[4]);
+ assertEquals(5, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveGenderstern() {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+
+ // Genderstern forms (these should already work via existing rules)
+ String[] tokens = tok.tokenize("Schüler*innen und Lehrer*innen");
+ assertEquals("Schüler*innen", tokens[0]);
+ assertEquals("und", tokens[1]);
+ assertEquals("Lehrer*innen", tokens[2]);
+ assertEquals(3, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveMixedSentence() {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+
+ // Mixed sentence with various gender forms
+ String[] tokens = tok.tokenize("Die Schüler:innen, Lehrer/innen und Mitarbeiter(innen) sowie Kaufmann/-frau trafen sich.");
+ assertEquals("Die", tokens[0]);
+ assertEquals("Schüler:innen", tokens[1]);
+ assertEquals(",", tokens[2]);
+ assertEquals("Lehrer/innen", tokens[3]);
+ assertEquals("und", tokens[4]);
+ assertEquals("Mitarbeiter(innen)", tokens[5]);
+ assertEquals("sowie", tokens[6]);
+ assertEquals("Kaufmann/-frau", tokens[7]);
+ assertEquals("trafen", tokens[8]);
+ assertEquals("sich", tokens[9]);
+ assertEquals(".", tokens[10]);
+ assertEquals(11, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveFormsNotRecognizedInEnglish() {
+ DerekoDfaTokenizer_en tok = new DerekoDfaTokenizer_en();
+
+ // English tokenizer should NOT recognize German gender-sensitive forms
+ // Colon forms should be separated
+ String[] tokens = tok.tokenize("Nutzer:innen and Teacher:in test");
+ assertEquals("Nutzer", tokens[0]);
+ assertEquals(":", tokens[1]);
+ assertEquals("innen", tokens[2]);
+
+ // Slash forms should be separated
+ tokens = tok.tokenize("Nutzer/innen Kaufmann/frau");
+ assertEquals("Nutzer", tokens[0]);
+ assertEquals("/", tokens[1]);
+ assertEquals("innen", tokens[2]);
+ assertEquals("Kaufmann", tokens[3]);
+ assertEquals("/", tokens[4]);
+ assertEquals("frau", tokens[5]);
+ assertEquals(6, tokens.length);
+ }
+
+ @Test
+ public void testGenderSensitiveFormsNotRecognizedInFrench() {
+ DerekoDfaTokenizer_fr tok = new DerekoDfaTokenizer_fr();
+
+ // French tokenizer should NOT recognize German gender-sensitive forms
+ // Colon forms should be separated
+ String[] tokens = tok.tokenize("Nutzer:innen et Teacher:in test");
+ assertEquals("Nutzer", tokens[0]);
+ assertEquals(":", tokens[1]);
+ assertEquals("innen", tokens[2]);
+
+ // Slash forms should be separated
+ tokens = tok.tokenize("Nutzer/innen Kaufmann/frau");
+ assertEquals("Nutzer", tokens[0]);
+ assertEquals("/", tokens[1]);
+ assertEquals("innen", tokens[2]);
+ assertEquals("Kaufmann", tokens[3]);
+ assertEquals("/", tokens[4]);
+ assertEquals("frau", tokens[5]);
+ assertEquals(6, tokens.length);
+ }
}