test/test.js

const { execSync } = require('child_process');

describe('conllu-gender', () => {

  test('Full mode: all gender-sensitive nouns (Genderstern singular)', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();

    // Genderstern singular noun: Bürger*in → lemma Bürger*in, NOUN NN Gender=Fem,Masc,NonBin|Number=Sing
    expect(stdout).toContain('Bürger*in\tBürger*in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Sing');
    // Long compound: Jugendpsychiater*in
    expect(stdout).toContain('Jugendpsychiater*in\tJugendpsychiater*in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Sing');
    // Compound with umlaut base: Generalstaatsanwält*in
    expect(stdout).toContain('Generalstaatsanwält*in\tGeneralstaatsanwält*in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Sing');
    // Nachfolger*in
    expect(stdout).toContain('Nachfolger*in\tNachfolger*in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Sing');
    // With umlaut base: Antragssteller*in (no umlaut but long compound)
    expect(stdout).toContain('Antragssteller*in\tAntragssteller*in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Sing');
  });

  test('Full mode: Genderstern plural nouns → lemma uses singular, Number=Plur', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    // Fachärzt*innen → lemma Fachärzt*in
    expect(stdout).toContain('Fachärzt*innen\tFachärzt*in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Plur');
  });

  test('Full mode: Doppelpunkt nouns', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    // Doppelpunkt singular
    expect(stdout).toContain('Anhänger:in\tAnhänger:in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Sing');
    expect(stdout).toContain('Wirt:in\tWirt:in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Sing');
    // Doppelpunkt plural  
    expect(stdout).toContain('Lehrer:innen\tLehrer:in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Plur');
    // Long compound with umlaut base
    expect(stdout).toContain('Menschenrechtsanwält:innen\tMenschenrechtsanwält:in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Plur');
  });

  test('Full mode: Unterstrich plural nouns', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    expect(stdout).toContain('Autor_innen\tAutor_in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Plur');
  });

  test('Full mode: Binnen-I nouns (binary intended → Gender=Masc,Fem)', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    // Binnen-I singular: ZeugIn → Gender=Masc,Fem|Number=Sing
    expect(stdout).toContain('ZeugIn\tZeugIn\tNOUN\tNN\tGender=Masc,Fem|Number=Sing');
    // Binnen-I singular: SchülerIn → Gender=Masc,Fem|Number=Sing
    expect(stdout).toContain('SchülerIn\tSchülerIn\tNOUN\tNN\tGender=Masc,Fem|Number=Sing');
    // Binnen-I plural: LehrerInnen → lemma LehrerIn, Number=Plur
    expect(stdout).toContain('LehrerInnen\tLehrerIn\tNOUN\tNN\tGender=Masc,Fem|Number=Plur');
  });

  test('Full mode: Klammern plural nouns (binary intended)', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    expect(stdout).toContain('Schüler(innen)\tSchüler(in)\tNOUN\tNN\tGender=Masc,Fem|Number=Plur');
  });

  test('Full mode: Schrägstrich nouns', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    // Simple slash
    expect(stdout).toContain('Autor/innen\tAutor/in\tNOUN\tNN\tGender=Masc,Fem|Number=Plur');
    // Slash with Ergänzungsstrich (/-innen)
    expect(stdout).toContain('Autor/-innen\tAutor/in\tNOUN\tNN\tGender=Masc,Fem|Number=Plur');
    expect(stdout).toContain('Spieler/-innen\tSpieler/in\tNOUN\tNN\tGender=Masc,Fem|Number=Plur');
  });

  test('Full mode: gendered determiners/pronouns (non-binary intended)', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    // jede*r → DET PIAT Gender=Fem,Masc,NonBin, lemma jede*r
    expect(stdout).toContain('jede*r\tjede*r\tDET\tPIAT\tGender=Fem,Masc,NonBin');
    const jeder_count = (stdout.match(/jede\*r\tjede\*r/g) || []).length;
    expect(jeder_count).toBe(2);  // appears in two sentences
    // eine*n → DET ART Gender=Fem,Masc,NonBin  (non-binary via *)
    expect(stdout).toContain('eine*n\teine*n\tDET\tART\tGender=Fem,Masc,NonBin');
    // jede:r → DET PIAT Gender=Fem,Masc,NonBin
    expect(stdout).toContain('jede:r\tjede:r\tDET\tPIAT\tGender=Fem,Masc,NonBin');
    // die*der → DET ART Gender=Fem,Masc,NonBin  (* marker → inclusive even for merged forms)
    expect(stdout).toContain('die*der\tdie*der\tDET\tART\tGender=Fem,Masc,NonBin');
  });

  test('Full mode: neo-pronoun (sie*er → PRON PPER Gender=Fem,Masc,NonBin)', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    expect(stdout).toContain('sie*er\tsie*er\tPRON\tPPER\tGender=Fem,Masc,NonBin');
  });

  test('Full mode: neo-pronoun lexicon forms (PRON PPER Gender=Fem,Masc,NonBin|PronType=Prs)', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    // sier: NOM of sier-paradigm
    expect(stdout).toContain('sier\tsier\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
    // xier: NOM of xier-paradigm
    expect(stdout).toContain('xier\txier\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
    // oj: NOM of oj-paradigm
    expect(stdout).toContain('oj\toj\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
    // el: NOM of el-paradigm
    expect(stdout).toContain('el\tel\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
    // hen: NOM of hen-paradigm
    expect(stdout).toContain('hen\then\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
  });

  test('Full mode: foundry comment changed to gender', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    const foundry_count = (stdout.match(/# foundry = gender/g) || []).length;
    expect(foundry_count).toBe(18);
  });

  test('Full mode: non-gender tokens pass through unchanged', () => {
    const command = 'node src/index.js < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    // Regular nouns are passed through, lemma stays _
    expect(stdout).toContain('Umbenennung\t_\tNOUN\tNN');
    expect(stdout).toContain('Ideen\t_\tNOUN\tNN');
  });

  test('Sparse mode: only annotated tokens are emitted', () => {
    const command = 'node src/index.js -s < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    const lines = stdout.split('\n');
    const tokenLines = lines.filter(l => l.match(/^\d+\t/));
    // Every token line must carry a gender annotation:
    // either the lemma column (col 2) is non-underscore (noun/pron was annotated)
    // or the features column (col 5) is non-underscore (det was annotated)
    tokenLines.forEach(line => {
      const cols = line.split('\t');
      const lemmaAnnotated  = cols[2] !== '_';
      const featsAnnotated  = cols[5] !== '_';
      expect(lemmaAnnotated || featsAnnotated).toBe(true);
    });
    // Count: 18 NOUN + 5 DET + 1 PRON (sie*er) + 5 neo-pronouns (sier,xier,oj,el,hen)
    //       + 7 new neo-pronouns from sentences 16–18 (el,em,ey,y,mensch,Mensch,xier) = 36
    expect(tokenLines.length).toBe(36);
  });

  test('Sparse mode: sentence headers are emitted for sentences with matches', () => {
    const command = 'node src/index.js -s < test/data/gender.conllu';
    const stdout = execSync(command).toString();
    // 12 original + 3 new sentences (16–18) have at least one gender form
    const text_id_count = (stdout.match(/# text_id = /g) || []).length;
    expect(text_id_count).toBe(15);
  });

  test('Inline input: basic Genderstern annotation', () => {
    const testInput = `# foundry = base
# text_id = inline-001
# text = Die Lehrerin und Lehrer*innen kamen
1\tDie\t_\tDET\tART\t_\t_\t_\t_\t_
2\tLehrerin\t_\tNOUN\tNN\t_\t_\t_\t_\t_
3\tund\t_\tCCONJ\tKON\t_\t_\t_\t_\t_
4\tLehrer*innen\t_\t_\t_\t_\t_\t_\t_\t_
5\tkamen\t_\tVERB\tVVFIN\t_\t_\t_\t_\t_

`;
    const stdout = execSync('node src/index.js', { input: testInput }).toString();
    expect(stdout).toContain('Lehrer*innen\tLehrer*in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Plur');
    // Regular noun 'Lehrerin' must not be incorrectly tagged
    expect(stdout).toContain('Lehrerin\t_\tNOUN\tNN\t_');
  });

  test('Inline input: existing POS/lemma/feats are replaced for gender forms', () => {
    const testInput = `# foundry = base
# text_id = inline-002
# text = jede Ärztin und jede*r Arzt*in
1\tjede\t_\tDET\tPIAT\tGender=Fem|Number=Sing\t_\t_\t_\t_
2\tÄrztin\tÄrztin\tNOUN\tNN\tGender=Fem|Number=Sing\t_\t_\t_\t_
3\tund\t_\tCCONJ\tKON\t_\t_\t_\t_\t_
4\tjede*r\t_\tDET\tPIAT\t_\t_\t_\t_\t_
5\tArzt*in\t_\t_\t_\t_\t_\t_\t_\t_

`;
    const stdout = execSync('node src/index.js', { input: testInput }).toString();
    // jede*r: missing feats should be filled in
    expect(stdout).toContain('jede*r\tjede*r\tDET\tPIAT\tGender=Fem,Masc,NonBin');
    // Arzt*in: umlaut base, missing everything
    expect(stdout).toContain('Arzt*in\tArzt*in\tNOUN\tNN\tGender=Fem,Masc,NonBin|Number=Sing');
    // Ärztin (regular moviertes Femininum, no gender marker): unchanged
    expect(stdout).toContain('Ärztin\tÄrztin\tNOUN\tNN\tGender=Fem|Number=Sing');
    // jede (without gender marker): unchanged
    expect(stdout).toContain('jede\t_\tDET\tPIAT\tGender=Fem|Number=Sing');
  });

  // ---------------------------------------------------------------------------
  // Regression tests: false-positive tokens that must NOT be tagged
  // ---------------------------------------------------------------------------

  test('No false positives: *, Y, per, EL, EM, Ey, sin mid-sentence pass through unchanged', () => {
    // Each of these appeared as spurious neo-pronoun matches in the original code.
    // They must not receive a neo-pronoun annotation.
    const testInput = `# foundry = base
# text_id = fp-001
# text = Hinweis auf * und Y sowie per Einschreiben
1\tHinweis\t_\tNOUN\tNN\t_\t_\t_\t_\t_
2\tauf\t_\tADP\tAPPR\t_\t_\t_\t_\t_
3\t*\t_\tPUNCT\t$(\t_\t_\t_\t_\t_
4\tund\t_\tCCONJ\tKON\t_\t_\t_\t_\t_
5\tY\t_\tNOUN\tNN\t_\t_\t_\t_\t_
6\tsowie\t_\tCCONJ\tKON\t_\t_\t_\t_\t_
7\tper\t_\tADP\tAPPR\t_\t_\t_\t_\t_
8\tEinschreiben\t_\tNOUN\tNN\t_\t_\t_\t_\t_

# foundry = base
# text_id = fp-002
# text = Verweise auf EL EM Ey sin im Text
1\tVerweise\t_\tNOUN\tNN\t_\t_\t_\t_\t_
2\tauf\t_\tADP\tAPPR\t_\t_\t_\t_\t_
3\tEL\t_\tNOUN\tNN\t_\t_\t_\t_\t_
4\tEM\t_\tNOUN\tNN\t_\t_\t_\t_\t_
5\tEy\t_\tITJ\tITJ\t_\t_\t_\t_\t_
6\tsin\t_\tNOUN\tNN\t_\t_\t_\t_\t_
7\tim\t_\tADP\tAPPRART\t_\t_\t_\t_\t_
8\tText\t_\tNOUN\tNN\t_\t_\t_\t_\t_

`;
    const stdout = execSync('node src/index.js', { input: testInput }).toString();
    // None of the false-positive tokens should receive a neo-pronoun annotation.
    // A token passes through unchanged when its lemma column stays '_' and
    // its upos/xpos/feats are not overwritten to PRON/PPER.
    expect(stdout).toContain('3\t*\t_\tPUNCT');
    expect(stdout).toContain('5\tY\t_\tNOUN');
    expect(stdout).toContain('7\tper\t_\tADP');
    expect(stdout).toContain('3\tEL\t_\tNOUN');
    expect(stdout).toContain('4\tEM\t_\tNOUN');
    expect(stdout).toContain('5\tEy\t_\tITJ');
    expect(stdout).toContain('6\tsin\t_\tNOUN');
  });

  test('No false positive: Mensch mid-sentence must not be tagged as neo-pronoun', () => {
    const testInput = `# foundry = base
# text_id = fp-003
# text = Jeder Mensch hat Würde
1\tJeder\t_\tDET\tPIAT\t_\t_\t_\t_\t_
2\tMensch\t_\tNOUN\tNN\t_\t_\t_\t_\t_
3\that\t_\tAUX\tVAFIN\t_\t_\t_\t_\t_
4\tWürde\t_\tNOUN\tNN\t_\t_\t_\t_\t_

`;
    const stdout = execSync('node src/index.js', { input: testInput }).toString();
    // 'Mensch' at position 2 (not sentence-initial) must not be tagged.
    expect(stdout).toContain('2\tMensch\t_\tNOUN\tNN\t_');
  });

  test('Neo-pronoun: lowercase el, em, ey, y mid-sentence are still tagged', () => {
    const testInput = `# foundry = base
# text_id = neo-lc-001
# text = dankte el und em für ey und y
1\tdankte\t_\tVERB\tVVFIN\t_\t_\t_\t_\t_
2\tel\t_\t_\t_\t_\t_\t_\t_\t_
3\tund\t_\tCCONJ\tKON\t_\t_\t_\t_\t_
4\tem\t_\t_\t_\t_\t_\t_\t_\t_
5\tfür\t_\tADP\tAPPR\t_\t_\t_\t_\t_
6\tey\t_\t_\t_\t_\t_\t_\t_\t_
7\tund\t_\tCCONJ\tKON\t_\t_\t_\t_\t_
8\ty\t_\t_\t_\t_\t_\t_\t_\t_

`;
    const stdout = execSync('node src/index.js', { input: testInput }).toString();
    expect(stdout).toContain('el\tel\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
    expect(stdout).toContain('em\tem\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
    expect(stdout).toContain('ey\tey\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
    expect(stdout).toContain('y\tY\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
  });

  test('Neo-pronoun: mensch lowercase and sentence-initial Mensch are tagged', () => {
    const testInput = `# foundry = base
# text_id = neo-mensch-001
# text = mensch fragte und Mensch antwortete
1\tmensch\t_\t_\t_\t_\t_\t_\t_\t_
2\tfragte\t_\tVERB\tVVFIN\t_\t_\t_\t_\t_
3\tund\t_\tCCONJ\tKON\t_\t_\t_\t_\t_
4\tMensch\t_\tNOUN\tNN\t_\t_\t_\t_\t_
5\tantwortete\t_\tVERB\tVVFIN\t_\t_\t_\t_\t_

# foundry = base
# text_id = neo-mensch-002
# text = Mensch traf xier
1\tMensch\t_\t_\t_\t_\t_\t_\t_\t_
2\ttraf\t_\tVERB\tVVFIN\t_\t_\t_\t_\t_
3\txier\t_\t_\t_\t_\t_\t_\t_\t_

`;
    const stdout = execSync('node src/index.js', { input: testInput }).toString();
    // lowercase 'mensch' → neo-pronoun
    expect(stdout).toContain('1\tmensch\tmensch\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
    // 'Mensch' mid-sentence (position 4) → unchanged common noun
    expect(stdout).toContain('4\tMensch\t_\tNOUN\tNN\t_');
    // sentence-initial 'Mensch' (position 1) → neo-pronoun
    expect(stdout).toContain('1\tMensch\tmensch\tPRON\tPPER\tGender=Fem,Masc,NonBin|PronType=Prs');
  });
});