blob: 567e80e171ec5975881ceeda3d25af59597a91a4 [file] [log] [blame]
#!/usr/bin/env node
// conllu-gender
// Reads CoNLL-U format from stdin and annotates German gender-sensitive personal
// nouns, gendered determiners/pronouns, and neo-pronouns with correct POS (UPOS
// and XPOS/STTS), lemma, and morphological features.
//
// Based on the morphosyntactic analysis in:
// Ochs, S. (2026). Die morphosyntaktische Integration neuer Gendersuffixe:
// Eine korpusbasierte Analyse deutschsprachiger Pressetexte.
// Gender Linguistics, 2. doi: https://doi.org/10.65020/0619d927
//
// Gender marker types (following Ochs & Rüdiger 2025):
// Non-binary intended: Genderstern (*), Doppelpunkt (:), Unterstrich (_)
// Binary intended: Binnen-I (I), Klammern (()), Schrägstrich (/)
// ---------------------------------------------------------------------------
// Regex patterns for gender-sensitive NOUNS
// ---------------------------------------------------------------------------
// Each regex captures: (base, marker, suffix)
// suffix is either 'in' (singular) or 'innen' (plural)
// Genderstern: Lehrer*in, Bürger*innen, Ärzt*innen
const nounGenderStarRegex = /^(.+)\*(in(?:nen)?)$/i;
// Doppelpunkt: Lehrer:in, Bürger:innen
const nounGenderColonRegex = /^(.+):(in(?:nen)?)$/i;
// Unterstrich: Lehrer_in, Bürger_innen
const nounGenderUnderscoreRegex = /^(.+)_(in(?:nen)?)$/i;
// Binnen-I: LehrerIn, LehrerInnen (case-sensitive – the I is uppercase)
// The base must end in a lowercase letter to avoid matching regular proper nouns
// that start a sentence. We require at least one lowercase letter before the I.
const nounBinnenIRegex = /^([A-ZÄÖÜ][a-zäöüß].*?[a-zäöüß])(In(?:nen)?)$/;
// Klammern: Lehrer(in), Lehrer(innen)
const nounKlammernRegex = /^(.+)\((in(?:nen)?)\)$/i;
// Schrägstrich: Lehrer/in, Lehrer/innen, Lehrer/-in, Lehrer/-innen
const nounSchraegstrichRegex = /^(.+)\/-?(in(?:nen)?)$/i;
// ---------------------------------------------------------------------------
// Regex patterns for gender-sensitive DETERMINERS / PRONOUNS
// (jede*r, ein*e, der*die, des*r, eines*r, etc.)
// ---------------------------------------------------------------------------
// Inflected forms of articles, indefinite articles, and pronouns with gender
// markers. Non-binary intended markers (*, :, _) are the most common.
// We match: any known determiner/pronoun stem + gender_marker + ending
// Gendered forms like: jede*r, jede:r, jede_r, kein*e, kein:e, ein*e, ein:e,
// ein_e, der*die, die*der, des*r, des*der, dem*der, den*die, etc.
// Strategy: match known Determiner/Pronoun base forms followed by gender marker
// and a short inflectional ending.
// Combined pattern: known pronoun/det base + non-binary marker + short ending
// This covers forms documented in Ochs (2026) §7.3.2–7.3.4
const detNonBinaryRegex = /^(jede[mn]?|jede[rs]?|keine?[mrns]?|eine?[mrns]?|de[mrns]|die|das|de[rs]|dem|den|aller?|manche[mrns]?|solche[mrns]?|welche[mrns]?|irgendeine[mrns]?)([*:_])([a-zäöürs]{1,3})$/i;
// Binnen-I variants of determiners: einE, jedeR, jedeN, JedeR, etc.
// Base (lowercase or title-case) + uppercase inflection letter(s)
const detBinnenIRegex = /^(jede[mn]?|keine?[mrns]?|eine?[mrns]?|alle?|manche?|solche?|welche?)([RNSEM]{1,2})$/;
// Doppelform determiners merged with Schrägstrich (the only binary-intended merge
// character for articles per Ochs 2026): ein/e, die/der, einen/r, etc.
// Non-binary markers (*, :, _) are handled by detNonBinaryRegex with Gender=NonBin.
const detDoppelformRegex = /^(der|die|das|dem|den|des|ein|eine|einen|einem|einer|eines)\/(der|die|das|dem|den|des|ein|eine|einen|einem|einer|eines|[rns])$/i;
// ---------------------------------------------------------------------------
// Neo-pronouns (new gender-neutral pronouns in German)
// ---------------------------------------------------------------------------
// Gendered-star pronoun pairs (sie*er, er*sie, ihr*sein, etc.)
const neopronGenderStarPairRegex = /^(sie|er|ihr|ihn?|ihm?|dich|sich|mich|mir|uns|euch|ihnen|seinen?|ihrem?|deren?|denen)([*:_])(sie|er|ihr|ihn?|ihm?|dich|sich|mich|mir|uns|euch|ihnen|seinen?|ihrem?|deren?|denen)$/i;
// ---------------------------------------------------------------------------
// Neo-pronoun lexicon (source: pronomen.net/beliebige:neopronomen)
// Maps lowercased surface form → { lemma, upos, xpos, feats }.
//
// Lemma: nominative form as listed on pronomen.net.
// UPOS: PRON | XPOS: PPER | FEATS: Gender=Fem,Masc,NonBin|PronType=Prs
//
// Excluded (too ambiguous with standard German words):
// 'dem' – dative definite article / demonstrative pronoun
// 'deren' – relative/demonstrative genitive pronoun
// 'denen' – relative/demonstrative dative pronoun
// 'per' – common German preposition
// 'pers' – excluded together with 'per'
//
// Shared/ambiguous oblique forms:
// 'sier','siem','sien' – NOM/DAT/ACC of sier-paradigm; also GEN/DAT/ACC of
// et/siem-paradigm (both annotated with lemma 'sier')
// 'em' – NOM of em/em-paradigm; also DAT of el/em and en/em
// 'ems' – GEN of both el/em and em/em (annotated as lemma 'em')
// 'en' – NOM/ACC/DAT of en/en; NOM/ACC of en/em (lemma 'en')
// 'ens' – GEN of en/em; also all forms of ens/ens (lemma 'ens')
// ---------------------------------------------------------------------------
function neoPron(lemma) {
return { lemma, upos: 'PRON', xpos: 'PPER', feats: 'Gender=Fem,Masc,NonBin|PronType=Prs' };
}
const NEO_PRONOUN_FORMS = new Map([
// ---- Verschmelzung (blend pronouns) ------------------------------------
// sier/siem (NOM=sier, GEN=sies, DAT=siem, ACC=sien)
['sier', neoPron('sier')],
['sies', neoPron('sier')],
['siem', neoPron('sier')],
['sien', neoPron('sier')],
// xier/xiem (NOM=xier, GEN=xies, DAT=xiem, ACC=xien)
['xier', neoPron('xier')],
['xies', neoPron('xier')],
['xiem', neoPron('xier')],
['xien', neoPron('xier')],
// ersie/ihmihr (NOM=ersie, GEN=seinihr, DAT=ihmihr, ACC=ihnsie)
['ersie', neoPron('ersie')],
['seinihr', neoPron('ersie')],
['ihmihr', neoPron('ersie')],
['ihnsie', neoPron('ersie')],
// ---- They-ähnlich (they-like pronouns) ---------------------------------
// dej/denen/dej (NOM=dej, GEN=deren, DAT=denen, ACC=dej)
// 'deren' and 'denen' omitted (overlap with standard German pronouns)
['dej', neoPron('dej')],
// dey/denen/dem and dey/denen/demm (NOM=dey; 'dem' excluded)
['dey', neoPron('dey')],
['demm', neoPron('dey')], // ACC of dey/denen/demm
// ey/emm (NOM=ey, GEN=eys, DAT=emm, ACC=emm)
['ey', neoPron('ey')],
['eys', neoPron('ey')],
['emm', neoPron('ey')],
// they/them (NOM=they, GEN=their, DAT=them, ACC=them)
['they', neoPron('they')],
['their', neoPron('they')],
['them', neoPron('they')],
// ---- Neuer Stamm (new-stem pronouns) -----------------------------------
// el/em (NOM=el, GEN=ems, DAT=em, ACC=en)
// 'ems' mapped to 'em'-paradigm below; 'em'/'en' mapped to their own NOM paradigms
['el', neoPron('el')],
// em/em (NOM=em, GEN=ems, DAT=em, ACC=em)
['em', neoPron('em')],
['ems', neoPron('em')], // GEN shared with el/em paradigm
// en/en (NOM=en, GEN=enses, DAT=en, ACC=en)
// en/em (NOM=en, GEN=ens, DAT=em, ACC=en) — DAT 'em' mapped to em-paradigm
['en', neoPron('en')],
['enses', neoPron('en')],
// ens/ens (NOM=ens, GEN=ens, DAT=ens, ACC=ens)
// 'ens' takes priority as NOM of ens-paradigm (also GEN of en/em)
['ens', neoPron('ens')],
// et/siem (NOM=et, GEN=sier, DAT=siem, ACC=sien)
// oblique forms 'sier'/'siem'/'sien' already mapped to sier-paradigm above
['et', neoPron('et')],
// ex/ex (all forms = ex)
['ex', neoPron('ex')],
// hän/sim (NOM=hän, GEN=sir, DAT=sim, ACC=sin)
['hän', neoPron('hän')],
['sir', neoPron('hän')],
['sim', neoPron('hän')],
['sin', neoPron('hän')],
// hen/hem (NOM=hen, GEN=hens, DAT=hem, ACC=hen)
['hen', neoPron('hen')],
['hens', neoPron('hen')],
['hem', neoPron('hen')],
// hie/hiem (NOM=hie, GEN=hein, DAT=hiem, ACC=hie)
['hie', neoPron('hie')],
['hein', neoPron('hie')],
['hiem', neoPron('hie')],
// iks/iks (NOM=iks, GEN=ikses, DAT=iks, ACC=iks)
['iks', neoPron('iks')],
['ikses', neoPron('iks')],
// ind/inde (NOM=ind, GEN=inds, DAT=inde, ACC=ind)
['ind', neoPron('ind')],
['inds', neoPron('ind')],
['inde', neoPron('ind')],
// mensch/mensch (NOM=mensch, GEN=menschs, DAT=mensch, ACC=mensch)
// Note: case-insensitive match means sentence-initial 'Mensch' (common noun)
// will also be tagged; acceptable in a gender-language–focused tagger.
['mensch', neoPron('mensch')],
['menschs', neoPron('mensch')],
// nin/nim (NOM=nin, GEN=nims, DAT=nim, ACC=nin)
['nin', neoPron('nin')],
['nims', neoPron('nin')],
['nim', neoPron('nin')],
// oj/ojm (NOM=oj, GEN=juj, DAT=ojm, ACC=ojn)
['oj', neoPron('oj')],
['juj', neoPron('oj')],
['ojm', neoPron('oj')],
['ojn', neoPron('oj')],
// per/per (all forms = per; GEN = pers)
// Note: 'per' also occurs as a German preposition (e.g. 'per E-Mail').
['per', neoPron('per')],
['pers', neoPron('per')],
// ser/sem (NOM=ser, GEN=ses, DAT=sem, ACC=sen)
['ser', neoPron('ser')],
['ses', neoPron('ser')],
['sem', neoPron('ser')],
['sen', neoPron('ser')],
// Y/Y (all forms = Y; GEN = Ys) — stored lowercase; lemma retains uppercase 'Y'
['y', neoPron('Y')],
['ys', neoPron('Y')],
// zet/zerm (NOM=zet, GEN=zets, DAT=zerm, ACC=zern)
['zet', neoPron('zet')],
['zets', neoPron('zet')],
['zerm', neoPron('zet')],
['zern', neoPron('zet')],
// */* (Stern; all forms = *; GEN = *s)
['*', neoPron('*')],
['*s', neoPron('*')],
]);
// ---------------------------------------------------------------------------
// Helpers
// ---------------------------------------------------------------------------
/**
* Determine if a suffix string represents singular or plural.
* 'in' (length 2) → Sing
* 'innen' (length 5) → Plur
* Works case-insensitively (In / Innen for Binnen-I forms).
*/
function getNumber(suffix) {
return /^innen$/i.test(suffix) ? 'Plur' : 'Sing';
}
/**
* Build the canonical lemma for a gendered noun.
* The lemma is always the nominative singular form, preserving the original
* gender marker. This follows the convention that the lemma reflects the
* citation form of the gendered derivate (Ochs 2026 §2).
*
* @param {string} base - derivation base (before the gender marker)
* @param {string} marker - gender marker character(s), e.g. '*', ':', '_', 'I',
* '(in)', '/in', etc.
* @param {string} markerType - 'star'|'colon'|'underscore'|'binnenI'|
* 'klammern'|'schraegstrich'
*/
function buildNounLemma(base, marker, markerType) {
switch (markerType) {
case 'star': return base + '*in';
case 'colon': return base + ':in';
case 'underscore': return base + '_in';
case 'binnenI': return base + 'In';
case 'klammern': return base + '(in)';
case 'schraegstrich':return base + '/in';
default: return base + marker + 'in';
}
}
/**
* Build the morphological features string for a gendered noun token.
* Per CoNLL-U conventions, features are sorted alphabetically by feature name.
*
* Gender values used (extending standard UD practice for German):
* NonBin – non-binary intended forms (*, :, _)
* Masc,Fem – binary inclusive forms (I, (), /)
*
* Case is not set here because it cannot be determined from surface form alone
* for the vast majority of gendered noun tokens (Ochs 2026 §7.1).
*
* @param {string} number - 'Sing' | 'Plur'
* @param {string} markerType - see buildNounLemma
*/
function buildNounFeatures(number, markerType) {
const genderIsNonBinary = ['star', 'colon', 'underscore'].includes(markerType);
const genderIsBinary = ['binnenI', 'klammern', 'schraegstrich'].includes(markerType);
const feats = [];
if (genderIsNonBinary) {
feats.push('Gender=Fem,Masc,NonBin');
} else if (genderIsBinary) {
feats.push('Gender=Masc,Fem');
}
feats.push('Number=' + number);
return feats.join('|');
}
// ---------------------------------------------------------------------------
// Command-line interface (mirrors conllu-cmc)
// ---------------------------------------------------------------------------
const optionDefinitions = [
{ name: 'sparse', alias: 's', type: Boolean,
description: 'Print only the tokens that received new annotations.' },
{ name: 'help', alias: 'h', type: Boolean,
description: 'Print this usage guide.' },
];
const sections = [
{
header: 'conllu-gender',
content: 'Reads CoNLL-U format from stdin and annotates German gender-sensitive ' +
'personal nouns, gendered determiners/pronouns, and neo-pronouns with ' +
'correct POS, lemma, and morphological features. Writes CoNLL-U to stdout.'
},
{
header: 'Synopsis',
content: '$ conllu-gender [-s] < input.conllu > output.conllu'
},
{
header: 'Options',
optionList: optionDefinitions
}
];
const getUsage = require('command-line-usage');
const commandLineArgs = require('command-line-args');
var options;
try {
options = commandLineArgs(optionDefinitions);
} catch (e) {
console.error(e.message);
options = { help: true };
}
if (options.help) {
const usage = getUsage(sections);
console.log(usage);
process.exit(0);
}
// ---------------------------------------------------------------------------
// CoNLL-U processing
// ---------------------------------------------------------------------------
const readline = require('readline');
global.header = '';
global.fileheader = '';
global.standalone = false;
const rl = readline.createInterface({
input: process.stdin,
output: process.stdout,
terminal: false,
});
/**
* Attempt to annotate a single CoNLL-U token (word form).
* Returns an annotation object on success, or null if the token is not a
* recognised gender-sensitive form.
*
* Annotation object shape:
* { lemma, upos, xpos, feats }
*/
function classifyToken(word) {
let m;
// ------------------------------------------------------------------
// 0. Neo-pronoun lexicon lookup (case-insensitive, exact form match)
// ------------------------------------------------------------------
const entry = NEO_PRONOUN_FORMS.get(word.toLowerCase());
if (entry) return entry;
// ------------------------------------------------------------------
// 1. Gender-sensitive NOUNS
// ------------------------------------------------------------------
// Genderstern (non-binary intended)
if ((m = nounGenderStarRegex.exec(word))) {
const [, base, suffix] = m;
const number = getNumber(suffix);
return {
lemma: buildNounLemma(base, '*', 'star'),
upos: 'NOUN',
xpos: 'NN',
feats: buildNounFeatures(number, 'star'),
};
}
// Doppelpunkt (non-binary intended)
if ((m = nounGenderColonRegex.exec(word))) {
const [, base, suffix] = m;
const number = getNumber(suffix);
return {
lemma: buildNounLemma(base, ':', 'colon'),
upos: 'NOUN',
xpos: 'NN',
feats: buildNounFeatures(number, 'colon'),
};
}
// Unterstrich (non-binary intended)
if ((m = nounGenderUnderscoreRegex.exec(word))) {
const [, base, suffix] = m;
const number = getNumber(suffix);
return {
lemma: buildNounLemma(base, '_', 'underscore'),
upos: 'NOUN',
xpos: 'NN',
feats: buildNounFeatures(number, 'underscore'),
};
}
// Schrägstrich (binary intended) – before Binnen-I to avoid false matches
if ((m = nounSchraegstrichRegex.exec(word))) {
const [, base, suffix] = m;
const number = getNumber(suffix);
return {
lemma: buildNounLemma(base, '/', 'schraegstrich'),
upos: 'NOUN',
xpos: 'NN',
feats: buildNounFeatures(number, 'schraegstrich'),
};
}
// Klammern (binary intended)
if ((m = nounKlammernRegex.exec(word))) {
const [, base, suffix] = m;
const number = getNumber(suffix);
return {
lemma: buildNounLemma(base, '()', 'klammern'),
upos: 'NOUN',
xpos: 'NN',
feats: buildNounFeatures(number, 'klammern'),
};
}
// Binnen-I (binary intended) – requires at least one lowercase letter before
// the I to distinguish from sentence-initial capitalisation
if ((m = nounBinnenIRegex.exec(word))) {
const [, base, suffix] = m;
const number = getNumber(suffix);
return {
lemma: buildNounLemma(base, 'I', 'binnenI'),
upos: 'NOUN',
xpos: 'NN',
feats: buildNounFeatures(number, 'binnenI'),
};
}
// ------------------------------------------------------------------
// 2. Gender-sensitive DETERMINERS / PRONOUNS
// ------------------------------------------------------------------
// Doppelform determiners merged with gender marker (der*die, des*r, etc.)
// Checked before detNonBinaryRegex because die*der is a Doppelform, not purely
// non-binary intended, and should receive Gender=Masc,Fem features.
if ((m = detDoppelformRegex.exec(word))) {
const [fullMatch, form1] = m;
return {
lemma: fullMatch,
upos: 'DET',
xpos: inferDetXpos(form1),
feats: 'Gender=Masc,Fem',
};
}
// Non-binary marker determiners (jede*r, ein:e, kein_e, etc.)
if ((m = detNonBinaryRegex.exec(word))) {
const [, detBase, marker, ending] = m;
// Preserve full base + marker + ending as lemma (no stripping needed;
// gendered determiners have no established uninflected citation form).
return {
lemma: detBase + marker + ending,
upos: 'DET',
xpos: inferDetXpos(detBase),
feats: 'Gender=Fem,Masc,NonBin',
};
}
// Binnen-I determiners (einE, JedeR, jedeN, etc.)
if ((m = detBinnenIRegex.exec(word))) {
const [, detBase, endings] = m;
return {
lemma: detBase + endings,
upos: 'DET',
xpos: inferDetXpos(detBase),
feats: 'Gender=Masc,Fem',
};
}
// ------------------------------------------------------------------
// 3. Neo-pronouns / gendered pronoun pairs
// ------------------------------------------------------------------
if ((m = neopronGenderStarPairRegex.exec(word))) {
const [fullMatch, pron1, marker, pron2] = m;
const markerType = marker === '*' ? 'star' : marker === ':' ? 'colon' : 'underscore';
return {
lemma: fullMatch,
upos: 'PRON',
xpos: inferPronXpos(pron1),
feats: markerType === 'star' || markerType === 'colon' || markerType === 'underscore'
? 'Gender=Fem,Masc,NonBin' : 'Gender=Masc,Fem',
};
}
return null;
}
/**
* Infer STTS XPOS tag for a determiner/article base.
*/
function inferDetXpos(base) {
const b = base.toLowerCase();
if (/^(der|die|das|de[mrns])/.test(b)) return 'ART';
if (/^(ein|eine|einen|einem|einer|eines|kein|keine|keinen|keinem|keiner|keines)/.test(b)) return 'ART';
if (/^(jede|jeder|jeden|jedem|jedes|jedem)/.test(b)) return 'PIAT';
if (/^(alle|aller|allen|alles|allem)/.test(b)) return 'PIAT';
if (/^(manche|mancher|manchen|manchem|manches)/.test(b)) return 'PIAT';
if (/^(solche|solcher|solchen|solchem|solches)/.test(b)) return 'PIAT';
if (/^(welche|welcher|welchen|welchem|welches)/.test(b)) return 'PWAT';
if (/^(irgend)/.test(b)) return 'PIAT';
return 'ART';
}
/**
* Infer STTS XPOS tag for a personal pronoun base.
*/
function inferPronXpos(base) {
const b = base.toLowerCase();
if (/^(ich|du|er|sie|es|wir|ihr|sie|mich|mir|dich|dir|sich|ihn|ihm|uns|euch)$/.test(b)) return 'PPER';
return 'PPER';
}
// ---------------------------------------------------------------------------
// Main line-by-line processing loop (mirrors conllu-cmc approach)
// ---------------------------------------------------------------------------
function parseConllu(line) {
// Handle foundry comment: change to 'gender'
if (line.match('#\\s*foundry')) {
if (line.match('=\\s*base')) {
if (options.sparse) {
global.standalone = true;
}
process.stdout.write('# foundry = gender\n');
} else {
process.stdout.write(`${line}\n`);
}
return;
}
if (global.standalone) {
if (line.match('^#\\s*filename')) {
global.fileheader = `${line}\n`;
return;
} else if (line.match('^#\\s*text_id')) {
global.fileheader += `${line}\n`;
return;
} else if (line.match('^#\\s*eo[ft]')) {
process.stdout.write(`${line}\n`);
return;
} else if (line.match('^#')) {
global.header += `${line}\n`;
return;
} else if (line.trim().match('^$')) {
if (global.header === '') {
process.stdout.write('\n');
}
global.header = '';
return;
}
} else {
if (!line.match('^\\d+')) {
process.stdout.write(`${line}\n`);
return;
}
}
const columns = line.trim().split('\t');
// CoNLL-U columns (0-indexed):
// 0:ID 1:FORM 2:LEMMA 3:UPOS 4:XPOS 5:FEATS 6:HEAD 7:DEPREL 8:DEPS 9:MISC
const word = columns[1];
const annotation = classifyToken(word);
if (annotation) {
// Replace lemma (col 2), UPOS (col 3), XPOS (col 4), FEATS (col 5)
columns[2] = annotation.lemma;
columns[3] = annotation.upos;
columns[4] = annotation.xpos;
columns[5] = annotation.feats;
if (global.standalone) {
process.stdout.write(global.fileheader);
process.stdout.write(global.header);
global.header = global.fileheader = '';
}
process.stdout.write(columns.join('\t') + '\n');
} else if (!global.standalone) {
process.stdout.write(`${line}\n`);
}
}
rl.on('line', parseConllu);
rl.on('close', () => process.exit(0));