Index extra inline w-attributes (norm, orig, phon, trans) as Krill layers
TEI-derived corpora can carry inline <w>-level annotations beyond pos and
lemma. Read norm, orig, phon and trans from the correspondingly named <f>
features in morpho.xml and emit each as its own Krill token layer
(foundry/<name>=tokens with foundry/<name>:<value> terms, original casing
preserved), grouped under the morpho foundry — just like pos (p) and lemma
(l). The set is intentionally restricted to these four names, and the
layer-detection scan is gated behind a single short-circuiting check, so
corpora without them pay at most one extra pass. The layers are Krill-only
and do not affect CoNLL-U output.
The four new MorphoSpan fields also had to be carried through the
field-selective copy/merge in collectKrillMorphoData{,Direct}, which
otherwise silently drops them from Krill while CoNLL-U still works.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Change-Id: I2695acd47f33de782045f4cee50799538e6a3c4b
diff --git a/CHANGELOG.md b/CHANGELOG.md
index a8020d3..3166c3e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,7 @@
- Krill `externalLink` is now derived for sources where the full-text link is not explicitly encoded as a `ref[@type=page_url]`/`link` element, mirroring the predecessor tool (KorAP-XML-Krill): DGD/AGD/FOLK transcripts (title `_DF_<n>` suffix stripped, title "DGD"), Süddeutsche Zeitung (`U<yy>` sigles → szarchiv link from the biblNote `ID:`, title "Süddeutsche Zeitung"), and Genios newspapers (corpus sigle → `orikuerzel` via a built-in mapping + biblNote `ID:` → genios.de link, title "GENIOS"). Explicitly encoded links still take precedence. Link titles are carried via a new `externalLinkTitle` helper field.
- Sentence segmentation fallback: when a corpus has no `s` spans in `structure.xml` (common for TEI conversions of custom corpora), sentence boundaries now fall back to other segment-like TEI elements, in order of preference: `posting` (chat/CMC), `l` (verse line), `seg` (segment), `u` (utterance). This makes integrated taggers/parsers and KorAP sentence-based queries work on such corpora.
- New `dck_sample.zip` test resource: two-text excerpt from the CC BY licensed Dortmunder Chat-Korpus with custom `cmc` tokenization/annotations (in one text the `s` spans are removed to exercise the sentence fallback)
+- Krill output now indexes extra inline `<w>`-level annotations from TEI-derived corpora as their own token layers, alongside POS (`p`) and lemma (`l`): `norm`, `orig`, `phon` and `trans` (read from the correspondingly named `<f>` features in `morpho.xml`). Each becomes a `foundry/<name>=tokens` layer with `foundry/<name>:<value>` terms (original casing preserved), grouped under the `morpho` foundry. The set is intentionally restricted to these four names to keep the Krill hot path fast. These layers are Krill-only and do not affect CoNLL-U output.
### Fixed
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index f339440..3cf08eb 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -4454,6 +4454,10 @@
"upos" -> currentSpan.upos = append(currentSpan.upos, value)
"xpos", "ctag", "pos" -> currentSpan.xpos = append(currentSpan.xpos, value.replace(UNKNOWN, "--"))
"feats", "msd" -> currentSpan.feats = append(currentSpan.feats, value)
+ "norm" -> currentSpan.norm = append(currentSpan.norm, value)
+ "orig" -> currentSpan.orig = append(currentSpan.orig, value)
+ "phon" -> currentSpan.phon = append(currentSpan.phon, value)
+ "trans" -> currentSpan.trans = append(currentSpan.trans, value)
"certainty" -> currentSpan.misc = append(currentSpan.misc, value)
}
}
@@ -4493,6 +4497,10 @@
"upos" -> fs.upos = append(fs.upos, value)
"xpos", "ctag", "pos" -> fs.xpos = append(fs.xpos, value.replace(UNKNOWN, "--"))
"feats", "msd" -> fs.feats = append(fs.feats, value)
+ "norm" -> fs.norm = append(fs.norm, value)
+ "orig" -> fs.orig = append(fs.orig, value)
+ "phon" -> fs.phon = append(fs.phon, value)
+ "trans" -> fs.trans = append(fs.trans, value)
"type" -> {
val typeVal = feature.getElementsByTagName("symbol").item(0).attributes.getNamedItem("value").textContent.trim()
fs.feats = append(fs.feats, typeVal)
@@ -4702,7 +4710,13 @@
var head: String? = "_",
var deprel: String? = "_",
var deps: String? = "_",
- var misc: String? = "_"
+ var misc: String? = "_",
+ // Extra inline w-attributes from TEI-derived corpora, emitted as their
+ // own Krill layers (norm, orig, phon, trans). Not used by CoNLL-U.
+ var norm: String? = "_",
+ var orig: String? = "_",
+ var phon: String? = "_",
+ var trans: String? = "_"
)
internal fun parseAndWriteAnnotatedConllu(annotatedConllu: String, task: AnnotationWorkerPool.AnnotationTask?) {
@@ -5890,6 +5904,10 @@
filteredSpan.xpos = span.xpos
filteredSpan.feats = span.feats
filteredSpan.misc = span.misc
+ filteredSpan.norm = span.norm
+ filteredSpan.orig = span.orig
+ filteredSpan.phon = span.phon
+ filteredSpan.trans = span.trans
} else if (annotationType == "dependency") {
// Copy only dependency annotations (head, deprel)
filteredSpan.head = span.head
@@ -5924,6 +5942,10 @@
if (newSpan.xpos != null && newSpan.xpos != "_" && (existingSpan.xpos == null || existingSpan.xpos == "_")) existingSpan.xpos = newSpan.xpos
if (newSpan.feats != null && newSpan.feats != "_" && (existingSpan.feats == null || existingSpan.feats == "_")) existingSpan.feats = newSpan.feats
if (newSpan.misc != null && newSpan.misc != "_" && (existingSpan.misc == null || existingSpan.misc == "_")) existingSpan.misc = newSpan.misc
+ if (newSpan.norm != null && newSpan.norm != "_" && (existingSpan.norm == null || existingSpan.norm == "_")) existingSpan.norm = newSpan.norm
+ if (newSpan.orig != null && newSpan.orig != "_" && (existingSpan.orig == null || existingSpan.orig == "_")) existingSpan.orig = newSpan.orig
+ if (newSpan.phon != null && newSpan.phon != "_" && (existingSpan.phon == null || existingSpan.phon == "_")) existingSpan.phon = newSpan.phon
+ if (newSpan.trans != null && newSpan.trans != "_" && (existingSpan.trans == null || existingSpan.trans == "_")) existingSpan.trans = newSpan.trans
}
mergedCount++
} else {
@@ -5965,6 +5987,10 @@
filteredSpan.xpos = span.xpos
filteredSpan.feats = span.feats
filteredSpan.misc = span.misc
+ filteredSpan.norm = span.norm
+ filteredSpan.orig = span.orig
+ filteredSpan.phon = span.phon
+ filteredSpan.trans = span.trans
} else if (annotationType == "dependency") {
// Copy only dependency annotations (head, deprel)
filteredSpan.head = span.head
@@ -5999,6 +6025,10 @@
if (newSpan.xpos != null && newSpan.xpos != "_" && (existingSpan.xpos == null || existingSpan.xpos == "_")) existingSpan.xpos = newSpan.xpos
if (newSpan.feats != null && newSpan.feats != "_" && (existingSpan.feats == null || existingSpan.feats == "_")) existingSpan.feats = newSpan.feats
if (newSpan.misc != null && newSpan.misc != "_" && (existingSpan.misc == null || existingSpan.misc == "_")) existingSpan.misc = newSpan.misc
+ if (newSpan.norm != null && newSpan.norm != "_" && (existingSpan.norm == null || existingSpan.norm == "_")) existingSpan.norm = newSpan.norm
+ if (newSpan.orig != null && newSpan.orig != "_" && (existingSpan.orig == null || existingSpan.orig == "_")) existingSpan.orig = newSpan.orig
+ if (newSpan.phon != null && newSpan.phon != "_" && (existingSpan.phon == null || existingSpan.phon == "_")) existingSpan.phon = newSpan.phon
+ if (newSpan.trans != null && newSpan.trans != "_" && (existingSpan.trans == null || existingSpan.trans == "_")) existingSpan.trans = newSpan.trans
}
mergedCount++
} else {
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
index e111fa7..76c6459 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
@@ -359,6 +359,20 @@
layers.add("u=tokens")
}
}
+
+ // Extra inline w-attribute layers (norm, orig, phon, trans) are
+ // rare; a single short-circuiting scan gates the four per-attribute
+ // checks so corpora without them pay at most one extra pass.
+ val hasExtra = morphoData?.any {
+ (it.norm != null && it.norm != "_") || (it.orig != null && it.orig != "_") ||
+ (it.phon != null && it.phon != "_") || (it.trans != null && it.trans != "_")
+ } ?: false
+ if (hasExtra) {
+ if (morphoData?.any { it.norm != null && it.norm != "_" } == true) layers.add("norm=tokens")
+ if (morphoData?.any { it.orig != null && it.orig != "_" } == true) layers.add("orig=tokens")
+ if (morphoData?.any { it.phon != null && it.phon != "_" } == true) layers.add("phon=tokens")
+ if (morphoData?.any { it.trans != null && it.trans != "_" } == true) layers.add("trans=tokens")
+ }
}
}
@@ -408,7 +422,9 @@
// Convert layer format: "d=rels" -> "dependency", "p=tokens" -> "morpho", etc.
val layerName = when {
layer.startsWith("d=") -> "dependency"
- layer.startsWith("l=") || layer.startsWith("p=") || layer.startsWith("m=") || layer.startsWith("u=") -> "morpho"
+ layer.startsWith("l=") || layer.startsWith("p=") || layer.startsWith("m=") || layer.startsWith("u=") ||
+ layer.startsWith("norm=") || layer.startsWith("orig=") ||
+ layer.startsWith("phon=") || layer.startsWith("trans=") -> "morpho"
else -> layer.split("=")[0]
}
val foundryLayer = "$foundryFullName/$layerName"
@@ -849,6 +865,29 @@
if (morphoSpan.upos != null && morphoSpan.upos != "_" && foundry != "tree_tagger") {
tokenAnnotations.add(jsonString("$prefix/u:${morphoSpan.upos!!.escapeKrillValue()}"))
}
+
+ // Extra inline w-attribute layers (norm, orig, phon, trans),
+ // each emitted under its own key like pos (p:) and lemma (l:).
+ if (morphoSpan.norm != null && morphoSpan.norm != "_") {
+ morphoSpan.norm!!.split("|").forEach {
+ tokenAnnotations.add(jsonString("$prefix/norm:${it.escapeKrillValue()}"))
+ }
+ }
+ if (morphoSpan.orig != null && morphoSpan.orig != "_") {
+ morphoSpan.orig!!.split("|").forEach {
+ tokenAnnotations.add(jsonString("$prefix/orig:${it.escapeKrillValue()}"))
+ }
+ }
+ if (morphoSpan.phon != null && morphoSpan.phon != "_") {
+ morphoSpan.phon!!.split("|").forEach {
+ tokenAnnotations.add(jsonString("$prefix/phon:${it.escapeKrillValue()}"))
+ }
+ }
+ if (morphoSpan.trans != null && morphoSpan.trans != "_") {
+ morphoSpan.trans!!.split("|").forEach {
+ tokenAnnotations.add(jsonString("$prefix/trans:${it.escapeKrillValue()}"))
+ }
+ }
}
// Dependency relations
diff --git a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/CustomTokenizationTest.kt b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/CustomTokenizationTest.kt
index f5f5a65..a21aba1 100644
--- a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/CustomTokenizationTest.kt
+++ b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/CustomTokenizationTest.kt
@@ -168,6 +168,83 @@
}
}
+ /**
+ * Rare TEI w-attributes (norm, orig, phon, trans) become their own Krill
+ * layers (like pos and lemma), but are not part of CoNLL-U. Here we inject
+ * all four into the first token of DCK/CPR/00001 to exercise the layers.
+ */
+ private fun dckSampleWithExtraAttributes(): File {
+ val baseZip = loadResource("dck_sample.zip").path
+ val outZipFile = File.createTempFile("dck_extraattr", ".zip")
+ java.util.zip.ZipOutputStream(outZipFile.outputStream()).use { outZip ->
+ java.util.zip.ZipFile(File(baseZip)).use { inZip ->
+ val entries = inZip.entries()
+ while (entries.hasMoreElements()) {
+ val entry = entries.nextElement()
+ outZip.putNextEntry(java.util.zip.ZipEntry(entry.name))
+ if (entry.name == "DCK/CPR/00001/cmc/morpho.xml") {
+ val content = inZip.getInputStream(entry).bufferedReader().use { it.readText() }
+ .replaceFirst(
+ "<f name=\"pos\">PPER</f>",
+ "<f name=\"pos\">PPER</f>" +
+ "<f name=\"norm\">wir</f><f name=\"orig\">Wir</f>" +
+ "<f name=\"phon\">viːɐ̯</f><f name=\"trans\">we</f>"
+ )
+ outZip.write(content.toByteArray())
+ } else {
+ inZip.getInputStream(entry).use { it.copyTo(outZip) }
+ }
+ outZip.closeEntry()
+ }
+ }
+ }
+ return outZipFile
+ }
+
+ @Test
+ fun conlluDoesNotLeakExtraAttributesIntoFeats() {
+ val zip = dckSampleWithExtraAttributes()
+ try {
+ assertEquals(0, debug(arrayOf("-q", zip.path)))
+ // The first token keeps an empty FEATS column; the extra w-attributes
+ // are Krill-only and must not appear in CoNLL-U.
+ val output = outContent.toString()
+ assertContains(output, "wir\twir\t_\tPPER\t_")
+ assertFalse(output.contains("norm="), "norm must not leak into CoNLL-U")
+ assertFalse(output.contains("orig="), "orig must not leak into CoNLL-U")
+ } finally {
+ zip.delete()
+ }
+ }
+
+ @Test
+ fun krillIndexesExtraAttributesAsOwnLayers() {
+ val zip = dckSampleWithExtraAttributes()
+ val outputDir = File.createTempFile("dck_extraattr_krill", "").apply {
+ delete()
+ mkdirs()
+ }
+ try {
+ assertEquals(0, debug(arrayOf("-t", "krill", "-q", "-D", outputDir.path, zip.path)))
+ val tar = File(outputDir, "${zip.nameWithoutExtension}.krill.tar")
+ assertTrue(tar.exists(), "Expected ${tar.name}")
+ val json1 = readKrillJsons(tar).getValue("DCK-CPR-00001.json")
+ // Each attribute is indexed under its own foundry/key (case preserved)
+ assertContains(json1, "\"cmc/norm:wir\"")
+ assertContains(json1, "\"cmc/orig:Wir\"")
+ assertContains(json1, "\"cmc/phon:viːɐ̯\"")
+ assertContains(json1, "\"cmc/trans:we\"")
+ // Layers are advertised in layerInfos
+ assertContains(json1, "cmc/norm=tokens")
+ assertContains(json1, "cmc/orig=tokens")
+ // A token without the extra attributes must not carry empty layers
+ assertFalse(json1.contains("\"cmc/norm:\""), "norm must only be emitted where present")
+ } finally {
+ zip.delete()
+ outputDir.deleteRecursively()
+ }
+ }
+
private fun readKrillJsons(tarFile: File): Map<String, String> {
val extractDir = File.createTempFile("krill_extract", "").let {
it.delete()