blob: 567e80e171ec5975881ceeda3d25af59597a91a4 [file] [log] [blame]
Marc Kupietzb777f9d2026-03-07 09:26:20 +01001#!/usr/bin/env node
2
3// conllu-gender
4// Reads CoNLL-U format from stdin and annotates German gender-sensitive personal
5// nouns, gendered determiners/pronouns, and neo-pronouns with correct POS (UPOS
6// and XPOS/STTS), lemma, and morphological features.
7//
8// Based on the morphosyntactic analysis in:
9// Ochs, S. (2026). Die morphosyntaktische Integration neuer Gendersuffixe:
10// Eine korpusbasierte Analyse deutschsprachiger Pressetexte.
11// Gender Linguistics, 2. doi: https://doi.org/10.65020/0619d927
12//
13// Gender marker types (following Ochs & Rüdiger 2025):
14// Non-binary intended: Genderstern (*), Doppelpunkt (:), Unterstrich (_)
15// Binary intended: Binnen-I (I), Klammern (()), Schrägstrich (/)
16
17// ---------------------------------------------------------------------------
18// Regex patterns for gender-sensitive NOUNS
19// ---------------------------------------------------------------------------
20// Each regex captures: (base, marker, suffix)
21// suffix is either 'in' (singular) or 'innen' (plural)
22
23// Genderstern: Lehrer*in, Bürger*innen, Ärzt*innen
24const nounGenderStarRegex = /^(.+)\*(in(?:nen)?)$/i;
25// Doppelpunkt: Lehrer:in, Bürger:innen
26const nounGenderColonRegex = /^(.+):(in(?:nen)?)$/i;
27// Unterstrich: Lehrer_in, Bürger_innen
28const nounGenderUnderscoreRegex = /^(.+)_(in(?:nen)?)$/i;
29// Binnen-I: LehrerIn, LehrerInnen (case-sensitive – the I is uppercase)
30// The base must end in a lowercase letter to avoid matching regular proper nouns
31// that start a sentence. We require at least one lowercase letter before the I.
32const nounBinnenIRegex = /^([A-ZÄÖÜ][a-zäöüß].*?[a-zäöüß])(In(?:nen)?)$/;
33// Klammern: Lehrer(in), Lehrer(innen)
34const nounKlammernRegex = /^(.+)\((in(?:nen)?)\)$/i;
35// Schrägstrich: Lehrer/in, Lehrer/innen, Lehrer/-in, Lehrer/-innen
36const nounSchraegstrichRegex = /^(.+)\/-?(in(?:nen)?)$/i;
37
38// ---------------------------------------------------------------------------
39// Regex patterns for gender-sensitive DETERMINERS / PRONOUNS
40// (jede*r, ein*e, der*die, des*r, eines*r, etc.)
41// ---------------------------------------------------------------------------
42// Inflected forms of articles, indefinite articles, and pronouns with gender
43// markers. Non-binary intended markers (*, :, _) are the most common.
44// We match: any known determiner/pronoun stem + gender_marker + ending
45
46// Gendered forms like: jede*r, jede:r, jede_r, kein*e, kein:e, ein*e, ein:e,
47// ein_e, der*die, die*der, des*r, des*der, dem*der, den*die, etc.
48// Strategy: match known Determiner/Pronoun base forms followed by gender marker
49// and a short inflectional ending.
50
51// Combined pattern: known pronoun/det base + non-binary marker + short ending
52// This covers forms documented in Ochs (2026) §7.3.2–7.3.4
53const detNonBinaryRegex = /^(jede[mn]?|jede[rs]?|keine?[mrns]?|eine?[mrns]?|de[mrns]|die|das|de[rs]|dem|den|aller?|manche[mrns]?|solche[mrns]?|welche[mrns]?|irgendeine[mrns]?)([*:_])([a-zäöürs]{1,3})$/i;
54
55// Binnen-I variants of determiners: einE, jedeR, jedeN, JedeR, etc.
56// Base (lowercase or title-case) + uppercase inflection letter(s)
57const detBinnenIRegex = /^(jede[mn]?|keine?[mrns]?|eine?[mrns]?|alle?|manche?|solche?|welche?)([RNSEM]{1,2})$/;
58
59// Doppelform determiners merged with Schrägstrich (the only binary-intended merge
60// character for articles per Ochs 2026): ein/e, die/der, einen/r, etc.
61// Non-binary markers (*, :, _) are handled by detNonBinaryRegex with Gender=NonBin.
62const detDoppelformRegex = /^(der|die|das|dem|den|des|ein|eine|einen|einem|einer|eines)\/(der|die|das|dem|den|des|ein|eine|einen|einem|einer|eines|[rns])$/i;
63
64// ---------------------------------------------------------------------------
65// Neo-pronouns (new gender-neutral pronouns in German)
66// ---------------------------------------------------------------------------
67// Gendered-star pronoun pairs (sie*er, er*sie, ihr*sein, etc.)
68const neopronGenderStarPairRegex = /^(sie|er|ihr|ihn?|ihm?|dich|sich|mich|mir|uns|euch|ihnen|seinen?|ihrem?|deren?|denen)([*:_])(sie|er|ihr|ihn?|ihm?|dich|sich|mich|mir|uns|euch|ihnen|seinen?|ihrem?|deren?|denen)$/i;
69
70// ---------------------------------------------------------------------------
Marc Kupietz1a9f16e2026-03-07 09:50:55 +010071// Neo-pronoun lexicon (source: pronomen.net/beliebige:neopronomen)
72// Maps lowercased surface form → { lemma, upos, xpos, feats }.
73//
74// Lemma: nominative form as listed on pronomen.net.
Marc Kupietzb2068f42026-03-07 21:58:05 +010075// UPOS: PRON | XPOS: PPER | FEATS: Gender=Fem,Masc,NonBin|PronType=Prs
Marc Kupietz1a9f16e2026-03-07 09:50:55 +010076//
77// Excluded (too ambiguous with standard German words):
78// 'dem' – dative definite article / demonstrative pronoun
79// 'deren' – relative/demonstrative genitive pronoun
80// 'denen' – relative/demonstrative dative pronoun
81// 'per' – common German preposition
82// 'pers' – excluded together with 'per'
83//
84// Shared/ambiguous oblique forms:
85// 'sier','siem','sien' – NOM/DAT/ACC of sier-paradigm; also GEN/DAT/ACC of
86// et/siem-paradigm (both annotated with lemma 'sier')
87// 'em' – NOM of em/em-paradigm; also DAT of el/em and en/em
88// 'ems' – GEN of both el/em and em/em (annotated as lemma 'em')
89// 'en' – NOM/ACC/DAT of en/en; NOM/ACC of en/em (lemma 'en')
90// 'ens' – GEN of en/em; also all forms of ens/ens (lemma 'ens')
91// ---------------------------------------------------------------------------
92
93function neoPron(lemma) {
Marc Kupietzb2068f42026-03-07 21:58:05 +010094 return { lemma, upos: 'PRON', xpos: 'PPER', feats: 'Gender=Fem,Masc,NonBin|PronType=Prs' };
Marc Kupietz1a9f16e2026-03-07 09:50:55 +010095}
96
97const NEO_PRONOUN_FORMS = new Map([
98 // ---- Verschmelzung (blend pronouns) ------------------------------------
99 // sier/siem (NOM=sier, GEN=sies, DAT=siem, ACC=sien)
100 ['sier', neoPron('sier')],
101 ['sies', neoPron('sier')],
102 ['siem', neoPron('sier')],
103 ['sien', neoPron('sier')],
104 // xier/xiem (NOM=xier, GEN=xies, DAT=xiem, ACC=xien)
105 ['xier', neoPron('xier')],
106 ['xies', neoPron('xier')],
107 ['xiem', neoPron('xier')],
108 ['xien', neoPron('xier')],
109 // ersie/ihmihr (NOM=ersie, GEN=seinihr, DAT=ihmihr, ACC=ihnsie)
110 ['ersie', neoPron('ersie')],
111 ['seinihr', neoPron('ersie')],
112 ['ihmihr', neoPron('ersie')],
113 ['ihnsie', neoPron('ersie')],
114
115 // ---- They-ähnlich (they-like pronouns) ---------------------------------
116 // dej/denen/dej (NOM=dej, GEN=deren, DAT=denen, ACC=dej)
117 // 'deren' and 'denen' omitted (overlap with standard German pronouns)
118 ['dej', neoPron('dej')],
119 // dey/denen/dem and dey/denen/demm (NOM=dey; 'dem' excluded)
120 ['dey', neoPron('dey')],
121 ['demm', neoPron('dey')], // ACC of dey/denen/demm
122 // ey/emm (NOM=ey, GEN=eys, DAT=emm, ACC=emm)
123 ['ey', neoPron('ey')],
124 ['eys', neoPron('ey')],
125 ['emm', neoPron('ey')],
126 // they/them (NOM=they, GEN=their, DAT=them, ACC=them)
127 ['they', neoPron('they')],
128 ['their', neoPron('they')],
129 ['them', neoPron('they')],
130
131 // ---- Neuer Stamm (new-stem pronouns) -----------------------------------
132 // el/em (NOM=el, GEN=ems, DAT=em, ACC=en)
133 // 'ems' mapped to 'em'-paradigm below; 'em'/'en' mapped to their own NOM paradigms
134 ['el', neoPron('el')],
135 // em/em (NOM=em, GEN=ems, DAT=em, ACC=em)
136 ['em', neoPron('em')],
137 ['ems', neoPron('em')], // GEN shared with el/em paradigm
138 // en/en (NOM=en, GEN=enses, DAT=en, ACC=en)
139 // en/em (NOM=en, GEN=ens, DAT=em, ACC=en) — DAT 'em' mapped to em-paradigm
140 ['en', neoPron('en')],
141 ['enses', neoPron('en')],
142 // ens/ens (NOM=ens, GEN=ens, DAT=ens, ACC=ens)
143 // 'ens' takes priority as NOM of ens-paradigm (also GEN of en/em)
144 ['ens', neoPron('ens')],
145 // et/siem (NOM=et, GEN=sier, DAT=siem, ACC=sien)
146 // oblique forms 'sier'/'siem'/'sien' already mapped to sier-paradigm above
147 ['et', neoPron('et')],
148 // ex/ex (all forms = ex)
149 ['ex', neoPron('ex')],
150 // hän/sim (NOM=hän, GEN=sir, DAT=sim, ACC=sin)
151 ['hän', neoPron('hän')],
152 ['sir', neoPron('hän')],
153 ['sim', neoPron('hän')],
154 ['sin', neoPron('hän')],
155 // hen/hem (NOM=hen, GEN=hens, DAT=hem, ACC=hen)
156 ['hen', neoPron('hen')],
157 ['hens', neoPron('hen')],
158 ['hem', neoPron('hen')],
159 // hie/hiem (NOM=hie, GEN=hein, DAT=hiem, ACC=hie)
160 ['hie', neoPron('hie')],
161 ['hein', neoPron('hie')],
162 ['hiem', neoPron('hie')],
163 // iks/iks (NOM=iks, GEN=ikses, DAT=iks, ACC=iks)
164 ['iks', neoPron('iks')],
165 ['ikses', neoPron('iks')],
166 // ind/inde (NOM=ind, GEN=inds, DAT=inde, ACC=ind)
167 ['ind', neoPron('ind')],
168 ['inds', neoPron('ind')],
169 ['inde', neoPron('ind')],
170 // mensch/mensch (NOM=mensch, GEN=menschs, DAT=mensch, ACC=mensch)
171 // Note: case-insensitive match means sentence-initial 'Mensch' (common noun)
172 // will also be tagged; acceptable in a gender-language–focused tagger.
173 ['mensch', neoPron('mensch')],
174 ['menschs', neoPron('mensch')],
175 // nin/nim (NOM=nin, GEN=nims, DAT=nim, ACC=nin)
176 ['nin', neoPron('nin')],
177 ['nims', neoPron('nin')],
178 ['nim', neoPron('nin')],
179 // oj/ojm (NOM=oj, GEN=juj, DAT=ojm, ACC=ojn)
180 ['oj', neoPron('oj')],
181 ['juj', neoPron('oj')],
182 ['ojm', neoPron('oj')],
183 ['ojn', neoPron('oj')],
184 // per/per (all forms = per; GEN = pers)
185 // Note: 'per' also occurs as a German preposition (e.g. 'per E-Mail').
186 ['per', neoPron('per')],
187 ['pers', neoPron('per')],
188 // ser/sem (NOM=ser, GEN=ses, DAT=sem, ACC=sen)
189 ['ser', neoPron('ser')],
190 ['ses', neoPron('ser')],
191 ['sem', neoPron('ser')],
192 ['sen', neoPron('ser')],
193 // Y/Y (all forms = Y; GEN = Ys) — stored lowercase; lemma retains uppercase 'Y'
194 ['y', neoPron('Y')],
195 ['ys', neoPron('Y')],
196 // zet/zerm (NOM=zet, GEN=zets, DAT=zerm, ACC=zern)
197 ['zet', neoPron('zet')],
198 ['zets', neoPron('zet')],
199 ['zerm', neoPron('zet')],
200 ['zern', neoPron('zet')],
201 // */* (Stern; all forms = *; GEN = *s)
202 ['*', neoPron('*')],
203 ['*s', neoPron('*')],
204]);
205
206// ---------------------------------------------------------------------------
Marc Kupietzb777f9d2026-03-07 09:26:20 +0100207// Helpers
208// ---------------------------------------------------------------------------
209
210/**
211 * Determine if a suffix string represents singular or plural.
212 * 'in' (length 2) → Sing
213 * 'innen' (length 5) → Plur
214 * Works case-insensitively (In / Innen for Binnen-I forms).
215 */
216function getNumber(suffix) {
217 return /^innen$/i.test(suffix) ? 'Plur' : 'Sing';
218}
219
220/**
221 * Build the canonical lemma for a gendered noun.
222 * The lemma is always the nominative singular form, preserving the original
223 * gender marker. This follows the convention that the lemma reflects the
224 * citation form of the gendered derivate (Ochs 2026 §2).
225 *
226 * @param {string} base - derivation base (before the gender marker)
227 * @param {string} marker - gender marker character(s), e.g. '*', ':', '_', 'I',
228 * '(in)', '/in', etc.
229 * @param {string} markerType - 'star'|'colon'|'underscore'|'binnenI'|
230 * 'klammern'|'schraegstrich'
231 */
232function buildNounLemma(base, marker, markerType) {
233 switch (markerType) {
234 case 'star': return base + '*in';
235 case 'colon': return base + ':in';
236 case 'underscore': return base + '_in';
237 case 'binnenI': return base + 'In';
238 case 'klammern': return base + '(in)';
239 case 'schraegstrich':return base + '/in';
240 default: return base + marker + 'in';
241 }
242}
243
244/**
245 * Build the morphological features string for a gendered noun token.
246 * Per CoNLL-U conventions, features are sorted alphabetically by feature name.
247 *
248 * Gender values used (extending standard UD practice for German):
249 * NonBin – non-binary intended forms (*, :, _)
250 * Masc,Fem – binary inclusive forms (I, (), /)
251 *
252 * Case is not set here because it cannot be determined from surface form alone
253 * for the vast majority of gendered noun tokens (Ochs 2026 §7.1).
254 *
255 * @param {string} number - 'Sing' | 'Plur'
256 * @param {string} markerType - see buildNounLemma
257 */
258function buildNounFeatures(number, markerType) {
259 const genderIsNonBinary = ['star', 'colon', 'underscore'].includes(markerType);
260 const genderIsBinary = ['binnenI', 'klammern', 'schraegstrich'].includes(markerType);
261
262 const feats = [];
263 if (genderIsNonBinary) {
Marc Kupietzb2068f42026-03-07 21:58:05 +0100264 feats.push('Gender=Fem,Masc,NonBin');
Marc Kupietzb777f9d2026-03-07 09:26:20 +0100265 } else if (genderIsBinary) {
266 feats.push('Gender=Masc,Fem');
267 }
268 feats.push('Number=' + number);
269 return feats.join('|');
270}
271
272// ---------------------------------------------------------------------------
273// Command-line interface (mirrors conllu-cmc)
274// ---------------------------------------------------------------------------
275
276const optionDefinitions = [
277 { name: 'sparse', alias: 's', type: Boolean,
278 description: 'Print only the tokens that received new annotations.' },
279 { name: 'help', alias: 'h', type: Boolean,
280 description: 'Print this usage guide.' },
281];
282
283const sections = [
284 {
285 header: 'conllu-gender',
286 content: 'Reads CoNLL-U format from stdin and annotates German gender-sensitive ' +
287 'personal nouns, gendered determiners/pronouns, and neo-pronouns with ' +
288 'correct POS, lemma, and morphological features. Writes CoNLL-U to stdout.'
289 },
290 {
291 header: 'Synopsis',
292 content: '$ conllu-gender [-s] < input.conllu > output.conllu'
293 },
294 {
295 header: 'Options',
296 optionList: optionDefinitions
297 }
298];
299
300const getUsage = require('command-line-usage');
301const commandLineArgs = require('command-line-args');
302
303var options;
304try {
305 options = commandLineArgs(optionDefinitions);
306} catch (e) {
307 console.error(e.message);
308 options = { help: true };
309}
310
311if (options.help) {
312 const usage = getUsage(sections);
313 console.log(usage);
314 process.exit(0);
315}
316
317// ---------------------------------------------------------------------------
318// CoNLL-U processing
319// ---------------------------------------------------------------------------
320
321const readline = require('readline');
322global.header = '';
323global.fileheader = '';
324global.standalone = false;
325
326const rl = readline.createInterface({
327 input: process.stdin,
328 output: process.stdout,
329 terminal: false,
330});
331
332/**
333 * Attempt to annotate a single CoNLL-U token (word form).
334 * Returns an annotation object on success, or null if the token is not a
335 * recognised gender-sensitive form.
336 *
337 * Annotation object shape:
338 * { lemma, upos, xpos, feats }
339 */
340function classifyToken(word) {
341 let m;
342
343 // ------------------------------------------------------------------
Marc Kupietz1a9f16e2026-03-07 09:50:55 +0100344 // 0. Neo-pronoun lexicon lookup (case-insensitive, exact form match)
345 // ------------------------------------------------------------------
346 const entry = NEO_PRONOUN_FORMS.get(word.toLowerCase());
347 if (entry) return entry;
348
349 // ------------------------------------------------------------------
Marc Kupietzb777f9d2026-03-07 09:26:20 +0100350 // 1. Gender-sensitive NOUNS
351 // ------------------------------------------------------------------
352
353 // Genderstern (non-binary intended)
354 if ((m = nounGenderStarRegex.exec(word))) {
355 const [, base, suffix] = m;
356 const number = getNumber(suffix);
357 return {
358 lemma: buildNounLemma(base, '*', 'star'),
359 upos: 'NOUN',
360 xpos: 'NN',
361 feats: buildNounFeatures(number, 'star'),
362 };
363 }
364
365 // Doppelpunkt (non-binary intended)
366 if ((m = nounGenderColonRegex.exec(word))) {
367 const [, base, suffix] = m;
368 const number = getNumber(suffix);
369 return {
370 lemma: buildNounLemma(base, ':', 'colon'),
371 upos: 'NOUN',
372 xpos: 'NN',
373 feats: buildNounFeatures(number, 'colon'),
374 };
375 }
376
377 // Unterstrich (non-binary intended)
378 if ((m = nounGenderUnderscoreRegex.exec(word))) {
379 const [, base, suffix] = m;
380 const number = getNumber(suffix);
381 return {
382 lemma: buildNounLemma(base, '_', 'underscore'),
383 upos: 'NOUN',
384 xpos: 'NN',
385 feats: buildNounFeatures(number, 'underscore'),
386 };
387 }
388
389 // Schrägstrich (binary intended) – before Binnen-I to avoid false matches
390 if ((m = nounSchraegstrichRegex.exec(word))) {
391 const [, base, suffix] = m;
392 const number = getNumber(suffix);
393 return {
394 lemma: buildNounLemma(base, '/', 'schraegstrich'),
395 upos: 'NOUN',
396 xpos: 'NN',
397 feats: buildNounFeatures(number, 'schraegstrich'),
398 };
399 }
400
401 // Klammern (binary intended)
402 if ((m = nounKlammernRegex.exec(word))) {
403 const [, base, suffix] = m;
404 const number = getNumber(suffix);
405 return {
406 lemma: buildNounLemma(base, '()', 'klammern'),
407 upos: 'NOUN',
408 xpos: 'NN',
409 feats: buildNounFeatures(number, 'klammern'),
410 };
411 }
412
413 // Binnen-I (binary intended) – requires at least one lowercase letter before
414 // the I to distinguish from sentence-initial capitalisation
415 if ((m = nounBinnenIRegex.exec(word))) {
416 const [, base, suffix] = m;
417 const number = getNumber(suffix);
418 return {
419 lemma: buildNounLemma(base, 'I', 'binnenI'),
420 upos: 'NOUN',
421 xpos: 'NN',
422 feats: buildNounFeatures(number, 'binnenI'),
423 };
424 }
425
426 // ------------------------------------------------------------------
427 // 2. Gender-sensitive DETERMINERS / PRONOUNS
428 // ------------------------------------------------------------------
429
430 // Doppelform determiners merged with gender marker (der*die, des*r, etc.)
431 // Checked before detNonBinaryRegex because die*der is a Doppelform, not purely
432 // non-binary intended, and should receive Gender=Masc,Fem features.
433 if ((m = detDoppelformRegex.exec(word))) {
434 const [fullMatch, form1] = m;
435 return {
436 lemma: fullMatch,
437 upos: 'DET',
438 xpos: inferDetXpos(form1),
439 feats: 'Gender=Masc,Fem',
440 };
441 }
442
443 // Non-binary marker determiners (jede*r, ein:e, kein_e, etc.)
444 if ((m = detNonBinaryRegex.exec(word))) {
445 const [, detBase, marker, ending] = m;
446 // Preserve full base + marker + ending as lemma (no stripping needed;
447 // gendered determiners have no established uninflected citation form).
448 return {
449 lemma: detBase + marker + ending,
450 upos: 'DET',
451 xpos: inferDetXpos(detBase),
Marc Kupietzb2068f42026-03-07 21:58:05 +0100452 feats: 'Gender=Fem,Masc,NonBin',
Marc Kupietzb777f9d2026-03-07 09:26:20 +0100453 };
454 }
455
456 // Binnen-I determiners (einE, JedeR, jedeN, etc.)
457 if ((m = detBinnenIRegex.exec(word))) {
458 const [, detBase, endings] = m;
459 return {
460 lemma: detBase + endings,
461 upos: 'DET',
462 xpos: inferDetXpos(detBase),
463 feats: 'Gender=Masc,Fem',
464 };
465 }
466
467 // ------------------------------------------------------------------
468 // 3. Neo-pronouns / gendered pronoun pairs
469 // ------------------------------------------------------------------
470
471 if ((m = neopronGenderStarPairRegex.exec(word))) {
472 const [fullMatch, pron1, marker, pron2] = m;
473 const markerType = marker === '*' ? 'star' : marker === ':' ? 'colon' : 'underscore';
474 return {
475 lemma: fullMatch,
476 upos: 'PRON',
477 xpos: inferPronXpos(pron1),
478 feats: markerType === 'star' || markerType === 'colon' || markerType === 'underscore'
Marc Kupietzb2068f42026-03-07 21:58:05 +0100479 ? 'Gender=Fem,Masc,NonBin' : 'Gender=Masc,Fem',
Marc Kupietzb777f9d2026-03-07 09:26:20 +0100480 };
481 }
482
483 return null;
484}
485
486/**
487 * Infer STTS XPOS tag for a determiner/article base.
488 */
489function inferDetXpos(base) {
490 const b = base.toLowerCase();
491 if (/^(der|die|das|de[mrns])/.test(b)) return 'ART';
492 if (/^(ein|eine|einen|einem|einer|eines|kein|keine|keinen|keinem|keiner|keines)/.test(b)) return 'ART';
493 if (/^(jede|jeder|jeden|jedem|jedes|jedem)/.test(b)) return 'PIAT';
494 if (/^(alle|aller|allen|alles|allem)/.test(b)) return 'PIAT';
495 if (/^(manche|mancher|manchen|manchem|manches)/.test(b)) return 'PIAT';
496 if (/^(solche|solcher|solchen|solchem|solches)/.test(b)) return 'PIAT';
497 if (/^(welche|welcher|welchen|welchem|welches)/.test(b)) return 'PWAT';
498 if (/^(irgend)/.test(b)) return 'PIAT';
499 return 'ART';
500}
501
502/**
503 * Infer STTS XPOS tag for a personal pronoun base.
504 */
505function inferPronXpos(base) {
506 const b = base.toLowerCase();
507 if (/^(ich|du|er|sie|es|wir|ihr|sie|mich|mir|dich|dir|sich|ihn|ihm|uns|euch)$/.test(b)) return 'PPER';
508 return 'PPER';
509}
510
511// ---------------------------------------------------------------------------
512// Main line-by-line processing loop (mirrors conllu-cmc approach)
513// ---------------------------------------------------------------------------
514
515function parseConllu(line) {
516 // Handle foundry comment: change to 'gender'
517 if (line.match('#\\s*foundry')) {
518 if (line.match('=\\s*base')) {
519 if (options.sparse) {
520 global.standalone = true;
521 }
522 process.stdout.write('# foundry = gender\n');
523 } else {
524 process.stdout.write(`${line}\n`);
525 }
526 return;
527 }
528
529 if (global.standalone) {
530 if (line.match('^#\\s*filename')) {
531 global.fileheader = `${line}\n`;
532 return;
533 } else if (line.match('^#\\s*text_id')) {
534 global.fileheader += `${line}\n`;
535 return;
536 } else if (line.match('^#\\s*eo[ft]')) {
537 process.stdout.write(`${line}\n`);
538 return;
539 } else if (line.match('^#')) {
540 global.header += `${line}\n`;
541 return;
542 } else if (line.trim().match('^$')) {
543 if (global.header === '') {
544 process.stdout.write('\n');
545 }
546 global.header = '';
547 return;
548 }
549 } else {
550 if (!line.match('^\\d+')) {
551 process.stdout.write(`${line}\n`);
552 return;
553 }
554 }
555
556 const columns = line.trim().split('\t');
557 // CoNLL-U columns (0-indexed):
558 // 0:ID 1:FORM 2:LEMMA 3:UPOS 4:XPOS 5:FEATS 6:HEAD 7:DEPREL 8:DEPS 9:MISC
559
560 const word = columns[1];
561 const annotation = classifyToken(word);
562
563 if (annotation) {
564 // Replace lemma (col 2), UPOS (col 3), XPOS (col 4), FEATS (col 5)
565 columns[2] = annotation.lemma;
566 columns[3] = annotation.upos;
567 columns[4] = annotation.xpos;
568 columns[5] = annotation.feats;
569
570 if (global.standalone) {
571 process.stdout.write(global.fileheader);
572 process.stdout.write(global.header);
573 global.header = global.fileheader = '';
574 }
575 process.stdout.write(columns.join('\t') + '\n');
576 } else if (!global.standalone) {
577 process.stdout.write(`${line}\n`);
578 }
579}
580
581rl.on('line', parseConllu);
582rl.on('close', () => process.exit(0));