src/index.js

#!/usr/bin/env node

// conllu-gender
// Reads CoNLL-U format from stdin and annotates German gender-sensitive personal
// nouns, gendered determiners/pronouns, and neo-pronouns with correct POS (UPOS
// and XPOS/STTS), lemma, and morphological features.
//
// Based on the morphosyntactic analysis in:
//   Ochs, S. (2026). Die morphosyntaktische Integration neuer Gendersuffixe:
//   Eine korpusbasierte Analyse deutschsprachiger Pressetexte.
//   Gender Linguistics, 2. doi: https://doi.org/10.65020/0619d927
//
// Gender marker types (following Ochs & Rüdiger 2025):
//   Non-binary intended: Genderstern (*), Doppelpunkt (:), Unterstrich (_)
//   Binary intended:     Binnen-I (I), Klammern (()), Schrägstrich (/)

// ---------------------------------------------------------------------------
// Regex patterns for gender-sensitive NOUNS
// ---------------------------------------------------------------------------
// Each regex captures: (base, marker, suffix)
// suffix is either 'in' (singular) or 'innen' (plural)

// Genderstern:  Lehrer*in, Bürger*innen, Ärzt*innen
const nounGenderStarRegex   = /^(.+)\*(in(?:nen)?)$/i;
// Doppelpunkt:  Lehrer:in, Bürger:innen
const nounGenderColonRegex  = /^(.+):(in(?:nen)?)$/i;
// Unterstrich:  Lehrer_in, Bürger_innen
const nounGenderUnderscoreRegex = /^(.+)_(in(?:nen)?)$/i;
// Binnen-I:     LehrerIn, LehrerInnen  (case-sensitive – the I is uppercase)
// The base must end in a lowercase letter to avoid matching regular proper nouns
// that start a sentence. We require at least one lowercase letter before the I.
const nounBinnenIRegex       = /^([A-ZÄÖÜ][a-zäöüß].*?[a-zäöüß])(In(?:nen)?)$/;
// Klammern:     Lehrer(in), Lehrer(innen)
const nounKlammernRegex     = /^(.+)\((in(?:nen)?)\)$/i;
// Schrägstrich: Lehrer/in, Lehrer/innen, Lehrer/-in, Lehrer/-innen
const nounSchraegstrichRegex = /^(.+)\/-?(in(?:nen)?)$/i;

// ---------------------------------------------------------------------------
// Regex patterns for gender-sensitive DETERMINERS / PRONOUNS
// (jede*r, ein*e, der*die, des*r, eines*r, etc.)
// ---------------------------------------------------------------------------
// Inflected forms of articles, indefinite articles, and pronouns with gender
// markers. Non-binary intended markers (*, :, _) are the most common.
// We match: any known determiner/pronoun stem + gender_marker + ending

// Gendered forms like: jede*r, jede:r, jede_r, kein*e, kein:e, ein*e, ein:e,
// ein_e, der*die, die*der, des*r, des*der, dem*der, den*die, etc.
// Strategy: match known Determiner/Pronoun base forms followed by gender marker
// and a short inflectional ending.

// Combined pattern: known pronoun/det base + non-binary marker + short ending
// This covers forms documented in Ochs (2026) §7.3.2–7.3.4
const detNonBinaryRegex = /^(jede[mn]?|jede[rs]?|keine?[mrns]?|eine?[mrns]?|de[mrns]|die|das|de[rs]|dem|den|aller?|manche[mrns]?|solche[mrns]?|welche[mrns]?|irgendeine[mrns]?)([*:_])([a-zäöürs]{1,3})$/i;

// Binnen-I variants of determiners: einE, jedeR, jedeN, JedeR, etc.
// Base (lowercase or title-case) + uppercase inflection letter(s)
const detBinnenIRegex = /^(jede[mn]?|keine?[mrns]?|eine?[mrns]?|alle?|manche?|solche?|welche?)([RNSEM]{1,2})$/;

// Doppelform determiners merged with Schrägstrich (the only binary-intended merge
// character for articles per Ochs 2026): ein/e, die/der, einen/r, etc.
// Non-binary markers (*, :, _) are handled by detNonBinaryRegex with Gender=NonBin.
const detDoppelformRegex = /^(der|die|das|dem|den|des|ein|eine|einen|einem|einer|eines)\/(der|die|das|dem|den|des|ein|eine|einen|einem|einer|eines|[rns])$/i;

// ---------------------------------------------------------------------------
// Neo-pronouns (new gender-neutral pronouns in German)
// ---------------------------------------------------------------------------
// Gendered-star pronoun pairs  (sie*er, er*sie, ihr*sein, etc.)
const neopronGenderStarPairRegex = /^(sie|er|ihr|ihn?|ihm?|dich|sich|mich|mir|uns|euch|ihnen|seinen?|ihrem?|deren?|denen)([*:_])(sie|er|ihr|ihn?|ihm?|dich|sich|mich|mir|uns|euch|ihnen|seinen?|ihrem?|deren?|denen)$/i;

// ---------------------------------------------------------------------------
// Neo-pronoun lexicon  (source: pronomen.net/beliebige:neopronomen)
// Maps lowercased surface form → { lemma, upos, xpos, feats }.
//
// Lemma: nominative form as listed on pronomen.net.
// UPOS:  PRON  |  XPOS: PPER  |  FEATS: Gender=NonBin|PronType=Prs
//
// Excluded (too ambiguous with standard German words):
//   'dem'   – dative definite article / demonstrative pronoun
//   'deren' – relative/demonstrative genitive pronoun
//   'denen' – relative/demonstrative dative pronoun
//   'per'   – common German preposition
//   'pers'  – excluded together with 'per'
//
// Shared/ambiguous oblique forms:
//   'sier','siem','sien' – NOM/DAT/ACC of sier-paradigm; also GEN/DAT/ACC of
//                          et/siem-paradigm (both annotated with lemma 'sier')
//   'em'   – NOM of em/em-paradigm; also DAT of el/em and en/em
//   'ems'  – GEN of both el/em and em/em (annotated as lemma 'em')
//   'en'   – NOM/ACC/DAT of en/en; NOM/ACC of en/em (lemma 'en')
//   'ens'  – GEN of en/em; also all forms of ens/ens (lemma 'ens')
// ---------------------------------------------------------------------------

function neoPron(lemma) {
  return { lemma, upos: 'PRON', xpos: 'PPER', feats: 'Gender=NonBin|PronType=Prs' };
}

const NEO_PRONOUN_FORMS = new Map([
  // ---- Verschmelzung (blend pronouns) ------------------------------------
  // sier/siem  (NOM=sier, GEN=sies, DAT=siem, ACC=sien)
  ['sier',    neoPron('sier')],
  ['sies',    neoPron('sier')],
  ['siem',    neoPron('sier')],
  ['sien',    neoPron('sier')],
  // xier/xiem  (NOM=xier, GEN=xies, DAT=xiem, ACC=xien)
  ['xier',    neoPron('xier')],
  ['xies',    neoPron('xier')],
  ['xiem',    neoPron('xier')],
  ['xien',    neoPron('xier')],
  // ersie/ihmihr  (NOM=ersie, GEN=seinihr, DAT=ihmihr, ACC=ihnsie)
  ['ersie',   neoPron('ersie')],
  ['seinihr', neoPron('ersie')],
  ['ihmihr',  neoPron('ersie')],
  ['ihnsie',  neoPron('ersie')],

  // ---- They-ähnlich (they-like pronouns) ---------------------------------
  // dej/denen/dej  (NOM=dej, GEN=deren, DAT=denen, ACC=dej)
  // 'deren' and 'denen' omitted (overlap with standard German pronouns)
  ['dej',     neoPron('dej')],
  // dey/denen/dem and dey/denen/demm  (NOM=dey; 'dem' excluded)
  ['dey',     neoPron('dey')],
  ['demm',    neoPron('dey')],   // ACC of dey/denen/demm
  // ey/emm  (NOM=ey, GEN=eys, DAT=emm, ACC=emm)
  ['ey',      neoPron('ey')],
  ['eys',     neoPron('ey')],
  ['emm',     neoPron('ey')],
  // they/them  (NOM=they, GEN=their, DAT=them, ACC=them)
  ['they',    neoPron('they')],
  ['their',   neoPron('they')],
  ['them',    neoPron('they')],

  // ---- Neuer Stamm (new-stem pronouns) -----------------------------------
  // el/em  (NOM=el, GEN=ems, DAT=em, ACC=en)
  // 'ems' mapped to 'em'-paradigm below; 'em'/'en' mapped to their own NOM paradigms
  ['el',      neoPron('el')],
  // em/em  (NOM=em, GEN=ems, DAT=em, ACC=em)
  ['em',      neoPron('em')],
  ['ems',     neoPron('em')],   // GEN shared with el/em paradigm
  // en/en  (NOM=en, GEN=enses, DAT=en, ACC=en)
  // en/em  (NOM=en, GEN=ens, DAT=em, ACC=en)  — DAT 'em' mapped to em-paradigm
  ['en',      neoPron('en')],
  ['enses',   neoPron('en')],
  // ens/ens  (NOM=ens, GEN=ens, DAT=ens, ACC=ens)
  // 'ens' takes priority as NOM of ens-paradigm (also GEN of en/em)
  ['ens',     neoPron('ens')],
  // et/siem  (NOM=et, GEN=sier, DAT=siem, ACC=sien)
  // oblique forms 'sier'/'siem'/'sien' already mapped to sier-paradigm above
  ['et',      neoPron('et')],
  // ex/ex  (all forms = ex)
  ['ex',      neoPron('ex')],
  // hän/sim  (NOM=hän, GEN=sir, DAT=sim, ACC=sin)
  ['hän',     neoPron('hän')],
  ['sir',     neoPron('hän')],
  ['sim',     neoPron('hän')],
  ['sin',     neoPron('hän')],
  // hen/hem  (NOM=hen, GEN=hens, DAT=hem, ACC=hen)
  ['hen',     neoPron('hen')],
  ['hens',    neoPron('hen')],
  ['hem',     neoPron('hen')],
  // hie/hiem  (NOM=hie, GEN=hein, DAT=hiem, ACC=hie)
  ['hie',     neoPron('hie')],
  ['hein',    neoPron('hie')],
  ['hiem',    neoPron('hie')],
  // iks/iks  (NOM=iks, GEN=ikses, DAT=iks, ACC=iks)
  ['iks',     neoPron('iks')],
  ['ikses',   neoPron('iks')],
  // ind/inde  (NOM=ind, GEN=inds, DAT=inde, ACC=ind)
  ['ind',     neoPron('ind')],
  ['inds',    neoPron('ind')],
  ['inde',    neoPron('ind')],
  // mensch/mensch  (NOM=mensch, GEN=menschs, DAT=mensch, ACC=mensch)
  // Note: case-insensitive match means sentence-initial 'Mensch' (common noun)
  // will also be tagged; acceptable in a gender-language–focused tagger.
  ['mensch',  neoPron('mensch')],
  ['menschs', neoPron('mensch')],
  // nin/nim  (NOM=nin, GEN=nims, DAT=nim, ACC=nin)
  ['nin',     neoPron('nin')],
  ['nims',    neoPron('nin')],
  ['nim',     neoPron('nin')],
  // oj/ojm  (NOM=oj, GEN=juj, DAT=ojm, ACC=ojn)
  ['oj',      neoPron('oj')],
  ['juj',     neoPron('oj')],
  ['ojm',     neoPron('oj')],
  ['ojn',     neoPron('oj')],
  // per/per  (all forms = per; GEN = pers)
  // Note: 'per' also occurs as a German preposition (e.g. 'per E-Mail').
  ['per',     neoPron('per')],
  ['pers',    neoPron('per')],
  // ser/sem  (NOM=ser, GEN=ses, DAT=sem, ACC=sen)
  ['ser',     neoPron('ser')],
  ['ses',     neoPron('ser')],
  ['sem',     neoPron('ser')],
  ['sen',     neoPron('ser')],
  // Y/Y  (all forms = Y; GEN = Ys) — stored lowercase; lemma retains uppercase 'Y'
  ['y',       neoPron('Y')],
  ['ys',      neoPron('Y')],
  // zet/zerm  (NOM=zet, GEN=zets, DAT=zerm, ACC=zern)
  ['zet',     neoPron('zet')],
  ['zets',    neoPron('zet')],
  ['zerm',    neoPron('zet')],
  ['zern',    neoPron('zet')],
  // */*  (Stern; all forms = *; GEN = *s)
  ['*',       neoPron('*')],
  ['*s',      neoPron('*')],
]);

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

/**
 * Determine if a suffix string represents singular or plural.
 * 'in'    (length 2) → Sing
 * 'innen' (length 5) → Plur
 * Works case-insensitively (In / Innen for Binnen-I forms).
 */
function getNumber(suffix) {
  return /^innen$/i.test(suffix) ? 'Plur' : 'Sing';
}

/**
 * Build the canonical lemma for a gendered noun.
 * The lemma is always the nominative singular form, preserving the original
 * gender marker. This follows the convention that the lemma reflects the
 * citation form of the gendered derivate (Ochs 2026 §2).
 *
 * @param {string} base   - derivation base (before the gender marker)
 * @param {string} marker - gender marker character(s), e.g. '*', ':', '_', 'I',
 *                          '(in)', '/in', etc.
 * @param {string} markerType - 'star'|'colon'|'underscore'|'binnenI'|
 *                              'klammern'|'schraegstrich'
 */
function buildNounLemma(base, marker, markerType) {
  switch (markerType) {
    case 'star':         return base + '*in';
    case 'colon':        return base + ':in';
    case 'underscore':   return base + '_in';
    case 'binnenI':      return base + 'In';
    case 'klammern':     return base + '(in)';
    case 'schraegstrich':return base + '/in';
    default:             return base + marker + 'in';
  }
}

/**
 * Build the morphological features string for a gendered noun token.
 * Per CoNLL-U conventions, features are sorted alphabetically by feature name.
 *
 * Gender values used (extending standard UD practice for German):
 *   NonBin  – non-binary intended forms (*, :, _)
 *   Masc,Fem – binary inclusive forms (I, (), /)
 *
 * Case is not set here because it cannot be determined from surface form alone
 * for the vast majority of gendered noun tokens (Ochs 2026 §7.1).
 *
 * @param {string} number     - 'Sing' | 'Plur'
 * @param {string} markerType - see buildNounLemma
 */
function buildNounFeatures(number, markerType) {
  const genderIsNonBinary = ['star', 'colon', 'underscore'].includes(markerType);
  const genderIsBinary    = ['binnenI', 'klammern', 'schraegstrich'].includes(markerType);

  const feats = [];
  if (genderIsNonBinary) {
    feats.push('Gender=NonBin');
  } else if (genderIsBinary) {
    feats.push('Gender=Masc,Fem');
  }
  feats.push('Number=' + number);
  return feats.join('|');
}

// ---------------------------------------------------------------------------
// Command-line interface  (mirrors conllu-cmc)
// ---------------------------------------------------------------------------

const optionDefinitions = [
  { name: 'sparse', alias: 's', type: Boolean,
    description: 'Print only the tokens that received new annotations.' },
  { name: 'help',   alias: 'h', type: Boolean,
    description: 'Print this usage guide.' },
];

const sections = [
  {
    header: 'conllu-gender',
    content: 'Reads CoNLL-U format from stdin and annotates German gender-sensitive ' +
             'personal nouns, gendered determiners/pronouns, and neo-pronouns with ' +
             'correct POS, lemma, and morphological features. Writes CoNLL-U to stdout.'
  },
  {
    header: 'Synopsis',
    content: '$ conllu-gender [-s] < input.conllu > output.conllu'
  },
  {
    header: 'Options',
    optionList: optionDefinitions
  }
];

const getUsage = require('command-line-usage');
const commandLineArgs = require('command-line-args');

var options;
try {
  options = commandLineArgs(optionDefinitions);
} catch (e) {
  console.error(e.message);
  options = { help: true };
}

if (options.help) {
  const usage = getUsage(sections);
  console.log(usage);
  process.exit(0);
}

// ---------------------------------------------------------------------------
// CoNLL-U processing
// ---------------------------------------------------------------------------

const readline = require('readline');
global.header = '';
global.fileheader = '';
global.standalone = false;

const rl = readline.createInterface({
  input: process.stdin,
  output: process.stdout,
  terminal: false,
});

/**
 * Attempt to annotate a single CoNLL-U token (word form).
 * Returns an annotation object on success, or null if the token is not a
 * recognised gender-sensitive form.
 *
 * Annotation object shape:
 *   { lemma, upos, xpos, feats }
 */
function classifyToken(word) {
  let m;

  // ------------------------------------------------------------------
  // 0. Neo-pronoun lexicon lookup (case-insensitive, exact form match)
  // ------------------------------------------------------------------
  const entry = NEO_PRONOUN_FORMS.get(word.toLowerCase());
  if (entry) return entry;

  // ------------------------------------------------------------------
  // 1. Gender-sensitive NOUNS
  // ------------------------------------------------------------------

  // Genderstern  (non-binary intended)
  if ((m = nounGenderStarRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, '*', 'star'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'star'),
    };
  }

  // Doppelpunkt  (non-binary intended)
  if ((m = nounGenderColonRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, ':', 'colon'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'colon'),
    };
  }

  // Unterstrich  (non-binary intended)
  if ((m = nounGenderUnderscoreRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, '_', 'underscore'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'underscore'),
    };
  }

  // Schrägstrich (binary intended) – before Binnen-I to avoid false matches
  if ((m = nounSchraegstrichRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, '/', 'schraegstrich'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'schraegstrich'),
    };
  }

  // Klammern  (binary intended)
  if ((m = nounKlammernRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, '()', 'klammern'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'klammern'),
    };
  }

  // Binnen-I (binary intended) – requires at least one lowercase letter before
  // the I to distinguish from sentence-initial capitalisation
  if ((m = nounBinnenIRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, 'I', 'binnenI'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'binnenI'),
    };
  }

  // ------------------------------------------------------------------
  // 2. Gender-sensitive DETERMINERS / PRONOUNS
  // ------------------------------------------------------------------

  // Doppelform determiners merged with gender marker (der*die, des*r, etc.)
  // Checked before detNonBinaryRegex because die*der is a Doppelform, not purely
  // non-binary intended, and should receive Gender=Masc,Fem features.
  if ((m = detDoppelformRegex.exec(word))) {
    const [fullMatch, form1] = m;
    return {
      lemma: fullMatch,
      upos:  'DET',
      xpos:  inferDetXpos(form1),
      feats: 'Gender=Masc,Fem',
    };
  }

  // Non-binary marker determiners (jede*r, ein:e, kein_e, etc.)
  if ((m = detNonBinaryRegex.exec(word))) {
    const [, detBase, marker, ending] = m;
    // Preserve full base + marker + ending as lemma (no stripping needed;
    // gendered determiners have no established uninflected citation form).
    return {
      lemma: detBase + marker + ending,
      upos:  'DET',
      xpos:  inferDetXpos(detBase),
      feats: 'Gender=NonBin',
    };
  }

  // Binnen-I determiners  (einE, JedeR, jedeN, etc.)
  if ((m = detBinnenIRegex.exec(word))) {
    const [, detBase, endings] = m;
    return {
      lemma: detBase + endings,
      upos:  'DET',
      xpos:  inferDetXpos(detBase),
      feats: 'Gender=Masc,Fem',
    };
  }

  // ------------------------------------------------------------------
  // 3. Neo-pronouns / gendered pronoun pairs
  // ------------------------------------------------------------------

  if ((m = neopronGenderStarPairRegex.exec(word))) {
    const [fullMatch, pron1, marker, pron2] = m;
    const markerType = marker === '*' ? 'star' : marker === ':' ? 'colon' : 'underscore';
    return {
      lemma: fullMatch,
      upos:  'PRON',
      xpos:  inferPronXpos(pron1),
      feats: markerType === 'star' || markerType === 'colon' || markerType === 'underscore'
             ? 'Gender=NonBin' : 'Gender=Masc,Fem',
    };
  }

  return null;
}

/**
 * Infer STTS XPOS tag for a determiner/article base.
 */
function inferDetXpos(base) {
  const b = base.toLowerCase();
  if (/^(der|die|das|de[mrns])/.test(b)) return 'ART';
  if (/^(ein|eine|einen|einem|einer|eines|kein|keine|keinen|keinem|keiner|keines)/.test(b)) return 'ART';
  if (/^(jede|jeder|jeden|jedem|jedes|jedem)/.test(b)) return 'PIAT';
  if (/^(alle|aller|allen|alles|allem)/.test(b)) return 'PIAT';
  if (/^(manche|mancher|manchen|manchem|manches)/.test(b)) return 'PIAT';
  if (/^(solche|solcher|solchen|solchem|solches)/.test(b)) return 'PIAT';
  if (/^(welche|welcher|welchen|welchem|welches)/.test(b)) return 'PWAT';
  if (/^(irgend)/.test(b)) return 'PIAT';
  return 'ART';
}

/**
 * Infer STTS XPOS tag for a personal pronoun base.
 */
function inferPronXpos(base) {
  const b = base.toLowerCase();
  if (/^(ich|du|er|sie|es|wir|ihr|sie|mich|mir|dich|dir|sich|ihn|ihm|uns|euch)$/.test(b)) return 'PPER';
  return 'PPER';
}

// ---------------------------------------------------------------------------
// Main line-by-line processing loop  (mirrors conllu-cmc approach)
// ---------------------------------------------------------------------------

function parseConllu(line) {
  // Handle foundry comment: change to 'gender'
  if (line.match('#\\s*foundry')) {
    if (line.match('=\\s*base')) {
      if (options.sparse) {
        global.standalone = true;
      }
      process.stdout.write('# foundry = gender\n');
    } else {
      process.stdout.write(`${line}\n`);
    }
    return;
  }

  if (global.standalone) {
    if (line.match('^#\\s*filename')) {
      global.fileheader = `${line}\n`;
      return;
    } else if (line.match('^#\\s*text_id')) {
      global.fileheader += `${line}\n`;
      return;
    } else if (line.match('^#\\s*eo[ft]')) {
      process.stdout.write(`${line}\n`);
      return;
    } else if (line.match('^#')) {
      global.header += `${line}\n`;
      return;
    } else if (line.trim().match('^$')) {
      if (global.header === '') {
        process.stdout.write('\n');
      }
      global.header = '';
      return;
    }
  } else {
    if (!line.match('^\\d+')) {
      process.stdout.write(`${line}\n`);
      return;
    }
  }

  const columns = line.trim().split('\t');
  // CoNLL-U columns (0-indexed):
  // 0:ID  1:FORM  2:LEMMA  3:UPOS  4:XPOS  5:FEATS  6:HEAD  7:DEPREL  8:DEPS  9:MISC

  const word = columns[1];
  const annotation = classifyToken(word);

  if (annotation) {
    // Replace lemma (col 2), UPOS (col 3), XPOS (col 4), FEATS (col 5)
    columns[2] = annotation.lemma;
    columns[3] = annotation.upos;
    columns[4] = annotation.xpos;
    columns[5] = annotation.feats;

    if (global.standalone) {
      process.stdout.write(global.fileheader);
      process.stdout.write(global.header);
      global.header = global.fileheader = '';
    }
    process.stdout.write(columns.join('\t') + '\n');
  } else if (!global.standalone) {
    process.stdout.write(`${line}\n`);
  }
}

rl.on('line', parseConllu);
rl.on('close', () => process.exit(0));