src/index.js

#!/usr/bin/env node

// conllu-gender
// Reads CoNLL-U format from stdin and annotates German gender-sensitive personal
// nouns, gendered determiners/pronouns, and neo-pronouns with correct POS (UPOS
// and XPOS/STTS), lemma, and morphological features.
//
// Based on the morphosyntactic analysis in:
//   Ochs, S. (2026). Die morphosyntaktische Integration neuer Gendersuffixe:
//   Eine korpusbasierte Analyse deutschsprachiger Pressetexte.
//   Gender Linguistics, 2. doi: https://doi.org/10.65020/0619d927
//
// Gender marker types (following Ochs & Rüdiger 2025):
//   Non-binary intended: Genderstern (*), Doppelpunkt (:), Unterstrich (_)
//   Binary intended:     Binnen-I (I), Klammern (()), Schrägstrich (/)

// ---------------------------------------------------------------------------
// Regex patterns for gender-sensitive NOUNS
// ---------------------------------------------------------------------------
// Each regex captures: (base, marker, suffix)
// suffix is either 'in' (singular) or 'innen' (plural)

// Genderstern:  Lehrer*in, Bürger*innen, Ärzt*innen
const nounGenderStarRegex   = /^(.+)\*(in(?:nen)?)$/i;
// Doppelpunkt:  Lehrer:in, Bürger:innen
const nounGenderColonRegex  = /^(.+):(in(?:nen)?)$/i;
// Unterstrich:  Lehrer_in, Bürger_innen
const nounGenderUnderscoreRegex = /^(.+)_(in(?:nen)?)$/i;
// Binnen-I:     LehrerIn, LehrerInnen  (case-sensitive – the I is uppercase)
// The base must end in a lowercase letter to avoid matching regular proper nouns
// that start a sentence. We require at least one lowercase letter before the I.
const nounBinnenIRegex       = /^([A-ZÄÖÜ][a-zäöüß].*?[a-zäöüß])(In(?:nen)?)$/;
// Klammern:     Lehrer(in), Lehrer(innen)
const nounKlammernRegex     = /^(.+)\((in(?:nen)?)\)$/i;
// Schrägstrich: Lehrer/in, Lehrer/innen, Lehrer/-in, Lehrer/-innen
const nounSchraegstrichRegex = /^(.+)\/-?(in(?:nen)?)$/i;

// ---------------------------------------------------------------------------
// Regex patterns for gender-sensitive DETERMINERS / PRONOUNS
// (jede*r, ein*e, der*die, des*r, eines*r, etc.)
// ---------------------------------------------------------------------------
// Inflected forms of articles, indefinite articles, and pronouns with gender
// markers. Non-binary intended markers (*, :, _) are the most common.
// We match: any known determiner/pronoun stem + gender_marker + ending

// Gendered forms like: jede*r, jede:r, jede_r, kein*e, kein:e, ein*e, ein:e,
// ein_e, der*die, die*der, des*r, des*der, dem*der, den*die, etc.
// Strategy: match known Determiner/Pronoun base forms followed by gender marker
// and a short inflectional ending.

// Combined pattern: known pronoun/det base + non-binary marker + short ending
// This covers forms documented in Ochs (2026) §7.3.2–7.3.4
const detNonBinaryRegex = /^(jede[mn]?|jede[rs]?|keine?[mrns]?|eine?[mrns]?|de[mrns]|die|das|de[rs]|dem|den|aller?|manche[mrns]?|solche[mrns]?|welche[mrns]?|irgendeine[mrns]?)([*:_])([a-zäöürs]{1,3})$/i;

// Binnen-I variants of determiners: einE, jedeR, jedeN, JedeR, etc.
// Base (lowercase or title-case) + uppercase inflection letter(s)
const detBinnenIRegex = /^(jede[mn]?|keine?[mrns]?|eine?[mrns]?|alle?|manche?|solche?|welche?)([RNSEM]{1,2})$/;

// Doppelform determiners merged with Schrägstrich (the only binary-intended merge
// character for articles per Ochs 2026): ein/e, die/der, einen/r, etc.
// Non-binary markers (*, :, _) are handled by detNonBinaryRegex with Gender=NonBin.
const detDoppelformRegex = /^(der|die|das|dem|den|des|ein|eine|einen|einem|einer|eines)\/(der|die|das|dem|den|des|ein|eine|einen|einem|einer|eines|[rns])$/i;

// ---------------------------------------------------------------------------
// Neo-pronouns (new gender-neutral pronouns in German)
// ---------------------------------------------------------------------------
// Gendered-star pronoun pairs  (sie*er, er*sie, ihr*sein, etc.)
const neopronGenderStarPairRegex = /^(sie|er|ihr|ihn?|ihm?|dich|sich|mich|mir|uns|euch|ihnen|seinen?|ihrem?|deren?|denen)([*:_])(sie|er|ihr|ihn?|ihm?|dich|sich|mich|mir|uns|euch|ihnen|seinen?|ihrem?|deren?|denen)$/i;

// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------

/**
 * Determine if a suffix string represents singular or plural.
 * 'in'    (length 2) → Sing
 * 'innen' (length 5) → Plur
 * Works case-insensitively (In / Innen for Binnen-I forms).
 */
function getNumber(suffix) {
  return /^innen$/i.test(suffix) ? 'Plur' : 'Sing';
}

/**
 * Build the canonical lemma for a gendered noun.
 * The lemma is always the nominative singular form, preserving the original
 * gender marker. This follows the convention that the lemma reflects the
 * citation form of the gendered derivate (Ochs 2026 §2).
 *
 * @param {string} base   - derivation base (before the gender marker)
 * @param {string} marker - gender marker character(s), e.g. '*', ':', '_', 'I',
 *                          '(in)', '/in', etc.
 * @param {string} markerType - 'star'|'colon'|'underscore'|'binnenI'|
 *                              'klammern'|'schraegstrich'
 */
function buildNounLemma(base, marker, markerType) {
  switch (markerType) {
    case 'star':         return base + '*in';
    case 'colon':        return base + ':in';
    case 'underscore':   return base + '_in';
    case 'binnenI':      return base + 'In';
    case 'klammern':     return base + '(in)';
    case 'schraegstrich':return base + '/in';
    default:             return base + marker + 'in';
  }
}

/**
 * Build the morphological features string for a gendered noun token.
 * Per CoNLL-U conventions, features are sorted alphabetically by feature name.
 *
 * Gender values used (extending standard UD practice for German):
 *   NonBin  – non-binary intended forms (*, :, _)
 *   Masc,Fem – binary inclusive forms (I, (), /)
 *
 * Case is not set here because it cannot be determined from surface form alone
 * for the vast majority of gendered noun tokens (Ochs 2026 §7.1).
 *
 * @param {string} number     - 'Sing' | 'Plur'
 * @param {string} markerType - see buildNounLemma
 */
function buildNounFeatures(number, markerType) {
  const genderIsNonBinary = ['star', 'colon', 'underscore'].includes(markerType);
  const genderIsBinary    = ['binnenI', 'klammern', 'schraegstrich'].includes(markerType);

  const feats = [];
  if (genderIsNonBinary) {
    feats.push('Gender=NonBin');
  } else if (genderIsBinary) {
    feats.push('Gender=Masc,Fem');
  }
  feats.push('Number=' + number);
  return feats.join('|');
}

// ---------------------------------------------------------------------------
// Command-line interface  (mirrors conllu-cmc)
// ---------------------------------------------------------------------------

const optionDefinitions = [
  { name: 'sparse', alias: 's', type: Boolean,
    description: 'Print only the tokens that received new annotations.' },
  { name: 'help',   alias: 'h', type: Boolean,
    description: 'Print this usage guide.' },
];

const sections = [
  {
    header: 'conllu-gender',
    content: 'Reads CoNLL-U format from stdin and annotates German gender-sensitive ' +
             'personal nouns, gendered determiners/pronouns, and neo-pronouns with ' +
             'correct POS, lemma, and morphological features. Writes CoNLL-U to stdout.'
  },
  {
    header: 'Synopsis',
    content: '$ conllu-gender [-s] < input.conllu > output.conllu'
  },
  {
    header: 'Options',
    optionList: optionDefinitions
  }
];

const getUsage = require('command-line-usage');
const commandLineArgs = require('command-line-args');

var options;
try {
  options = commandLineArgs(optionDefinitions);
} catch (e) {
  console.error(e.message);
  options = { help: true };
}

if (options.help) {
  const usage = getUsage(sections);
  console.log(usage);
  process.exit(0);
}

// ---------------------------------------------------------------------------
// CoNLL-U processing
// ---------------------------------------------------------------------------

const readline = require('readline');
global.header = '';
global.fileheader = '';
global.standalone = false;

const rl = readline.createInterface({
  input: process.stdin,
  output: process.stdout,
  terminal: false,
});

/**
 * Attempt to annotate a single CoNLL-U token (word form).
 * Returns an annotation object on success, or null if the token is not a
 * recognised gender-sensitive form.
 *
 * Annotation object shape:
 *   { lemma, upos, xpos, feats }
 */
function classifyToken(word) {
  let m;

  // ------------------------------------------------------------------
  // 1. Gender-sensitive NOUNS
  // ------------------------------------------------------------------

  // Genderstern  (non-binary intended)
  if ((m = nounGenderStarRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, '*', 'star'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'star'),
    };
  }

  // Doppelpunkt  (non-binary intended)
  if ((m = nounGenderColonRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, ':', 'colon'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'colon'),
    };
  }

  // Unterstrich  (non-binary intended)
  if ((m = nounGenderUnderscoreRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, '_', 'underscore'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'underscore'),
    };
  }

  // Schrägstrich (binary intended) – before Binnen-I to avoid false matches
  if ((m = nounSchraegstrichRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, '/', 'schraegstrich'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'schraegstrich'),
    };
  }

  // Klammern  (binary intended)
  if ((m = nounKlammernRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, '()', 'klammern'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'klammern'),
    };
  }

  // Binnen-I (binary intended) – requires at least one lowercase letter before
  // the I to distinguish from sentence-initial capitalisation
  if ((m = nounBinnenIRegex.exec(word))) {
    const [, base, suffix] = m;
    const number = getNumber(suffix);
    return {
      lemma: buildNounLemma(base, 'I', 'binnenI'),
      upos:  'NOUN',
      xpos:  'NN',
      feats: buildNounFeatures(number, 'binnenI'),
    };
  }

  // ------------------------------------------------------------------
  // 2. Gender-sensitive DETERMINERS / PRONOUNS
  // ------------------------------------------------------------------

  // Doppelform determiners merged with gender marker (der*die, des*r, etc.)
  // Checked before detNonBinaryRegex because die*der is a Doppelform, not purely
  // non-binary intended, and should receive Gender=Masc,Fem features.
  if ((m = detDoppelformRegex.exec(word))) {
    const [fullMatch, form1] = m;
    return {
      lemma: fullMatch,
      upos:  'DET',
      xpos:  inferDetXpos(form1),
      feats: 'Gender=Masc,Fem',
    };
  }

  // Non-binary marker determiners (jede*r, ein:e, kein_e, etc.)
  if ((m = detNonBinaryRegex.exec(word))) {
    const [, detBase, marker, ending] = m;
    // Preserve full base + marker + ending as lemma (no stripping needed;
    // gendered determiners have no established uninflected citation form).
    return {
      lemma: detBase + marker + ending,
      upos:  'DET',
      xpos:  inferDetXpos(detBase),
      feats: 'Gender=NonBin',
    };
  }

  // Binnen-I determiners  (einE, JedeR, jedeN, etc.)
  if ((m = detBinnenIRegex.exec(word))) {
    const [, detBase, endings] = m;
    return {
      lemma: detBase + endings,
      upos:  'DET',
      xpos:  inferDetXpos(detBase),
      feats: 'Gender=Masc,Fem',
    };
  }

  // ------------------------------------------------------------------
  // 3. Neo-pronouns / gendered pronoun pairs
  // ------------------------------------------------------------------

  if ((m = neopronGenderStarPairRegex.exec(word))) {
    const [fullMatch, pron1, marker, pron2] = m;
    const markerType = marker === '*' ? 'star' : marker === ':' ? 'colon' : 'underscore';
    return {
      lemma: fullMatch,
      upos:  'PRON',
      xpos:  inferPronXpos(pron1),
      feats: markerType === 'star' || markerType === 'colon' || markerType === 'underscore'
             ? 'Gender=NonBin' : 'Gender=Masc,Fem',
    };
  }

  return null;
}

/**
 * Infer STTS XPOS tag for a determiner/article base.
 */
function inferDetXpos(base) {
  const b = base.toLowerCase();
  if (/^(der|die|das|de[mrns])/.test(b)) return 'ART';
  if (/^(ein|eine|einen|einem|einer|eines|kein|keine|keinen|keinem|keiner|keines)/.test(b)) return 'ART';
  if (/^(jede|jeder|jeden|jedem|jedes|jedem)/.test(b)) return 'PIAT';
  if (/^(alle|aller|allen|alles|allem)/.test(b)) return 'PIAT';
  if (/^(manche|mancher|manchen|manchem|manches)/.test(b)) return 'PIAT';
  if (/^(solche|solcher|solchen|solchem|solches)/.test(b)) return 'PIAT';
  if (/^(welche|welcher|welchen|welchem|welches)/.test(b)) return 'PWAT';
  if (/^(irgend)/.test(b)) return 'PIAT';
  return 'ART';
}

/**
 * Infer STTS XPOS tag for a personal pronoun base.
 */
function inferPronXpos(base) {
  const b = base.toLowerCase();
  if (/^(ich|du|er|sie|es|wir|ihr|sie|mich|mir|dich|dir|sich|ihn|ihm|uns|euch)$/.test(b)) return 'PPER';
  return 'PPER';
}

// ---------------------------------------------------------------------------
// Main line-by-line processing loop  (mirrors conllu-cmc approach)
// ---------------------------------------------------------------------------

function parseConllu(line) {
  // Handle foundry comment: change to 'gender'
  if (line.match('#\\s*foundry')) {
    if (line.match('=\\s*base')) {
      if (options.sparse) {
        global.standalone = true;
      }
      process.stdout.write('# foundry = gender\n');
    } else {
      process.stdout.write(`${line}\n`);
    }
    return;
  }

  if (global.standalone) {
    if (line.match('^#\\s*filename')) {
      global.fileheader = `${line}\n`;
      return;
    } else if (line.match('^#\\s*text_id')) {
      global.fileheader += `${line}\n`;
      return;
    } else if (line.match('^#\\s*eo[ft]')) {
      process.stdout.write(`${line}\n`);
      return;
    } else if (line.match('^#')) {
      global.header += `${line}\n`;
      return;
    } else if (line.trim().match('^$')) {
      if (global.header === '') {
        process.stdout.write('\n');
      }
      global.header = '';
      return;
    }
  } else {
    if (!line.match('^\\d+')) {
      process.stdout.write(`${line}\n`);
      return;
    }
  }

  const columns = line.trim().split('\t');
  // CoNLL-U columns (0-indexed):
  // 0:ID  1:FORM  2:LEMMA  3:UPOS  4:XPOS  5:FEATS  6:HEAD  7:DEPREL  8:DEPS  9:MISC

  const word = columns[1];
  const annotation = classifyToken(word);

  if (annotation) {
    // Replace lemma (col 2), UPOS (col 3), XPOS (col 4), FEATS (col 5)
    columns[2] = annotation.lemma;
    columns[3] = annotation.upos;
    columns[4] = annotation.xpos;
    columns[5] = annotation.feats;

    if (global.standalone) {
      process.stdout.write(global.fileheader);
      process.stdout.write(global.header);
      global.header = global.fileheader = '';
    }
    process.stdout.write(columns.join('\t') + '\n');
  } else if (!global.standalone) {
    process.stdout.write(`${line}\n`);
  }
}

rl.on('line', parseConllu);
rl.on('close', () => process.exit(0));