Add stand-off metadata support for Krill output
Collect text-level metadata from stand-off XML files (one file per corpus,
in the <standOff> form produced e.g. by the wiki-taxonomy classifier) and
emit it as Krill metadata fields, joined to texts by raw_text/@docid.
- StandoffMetadata: auto-detects inputs by content (root element <standOff>),
so no CLI option is needed; filenames are cosmetic. Classification layers
become type:keywords fields named after the layer xml:id; links layers
become type:attachement (KorAP-link encoding).
- By default ingest everything in the file (the annotation tool already
applied its own top-k/threshold). The top-n/min-cert knobs are kept as
constants/params for a future --standoff-select option.
- KrillJsonGenerator: new StandoffField type and KrillTextData.standoffFields,
rendered after header fields; never clobbers an existing header field key.
- KorapXmlTool: detects and strips stand-off XML inputs from the ZIP list,
parses them (krill only; warns otherwise), and injects per text.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Change-Id: Ibd50d9393c052ab8172d1e41759b19ce47380e2c
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index 6542128..bdd01d2 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -811,6 +811,19 @@
}
}
+ // Auto-detect stand-off metadata inputs (<standOff> XML files) among the
+ // positional arguments, so no dedicated CLI option is needed. They are
+ // joined to texts by docid, so any number of files can be supplied and
+ // their filenames don't matter. Strip them from the ZIP processing list.
+ zipFileNames?.filter { StandoffMetadata.isStandoffMetadataFile(it) }?.takeIf { it.isNotEmpty() }?.let { standoffFiles ->
+ zipFileNames = zipFileNames!!.filterNot { it in standoffFiles }.toTypedArray()
+ if (outputFormat == OutputFormat.KRILL) {
+ standoffFiles.forEach { StandoffMetadata.parseInto(it, standoffMetadata) }
+ } else {
+ LOGGER.warning("Stand-off metadata files are only used with -t krill; ignoring: ${standoffFiles.joinToString(", ")}")
+ }
+ }
+
// For krill format, redirect logging to file before any logging occurs
if (outputFormat == OutputFormat.KRILL) {
// Determine output path for Krill format
@@ -1262,6 +1275,9 @@
)
val krillData: ConcurrentHashMap<String, KrillJsonGenerator.KrillTextData> = ConcurrentHashMap()
+ // Stand-off metadata fields keyed by text docid, loaded from <standOff> XML
+ // inputs and merged into each text's Krill fields at output time.
+ val standoffMetadata: MutableMap<String, MutableList<KrillJsonGenerator.StandoffField>> = HashMap()
val krillCompressedData: ConcurrentHashMap<String, CompressedKrillData> = ConcurrentHashMap()
val krillCompressionFutures: ConcurrentHashMap<String, java.util.concurrent.Future<*>> = ConcurrentHashMap()
val krillCompressionStartNanos: ConcurrentHashMap<String, Long> = ConcurrentHashMap()
@@ -6012,6 +6028,13 @@
)
textData.headerMetadata.clear()
textData.headerMetadata.putAll(resolvedMetadata)
+
+ // Attach any stand-off metadata fields for this text (joined by docid).
+ if (standoffMetadata.isNotEmpty()) {
+ standoffMetadata[textId]?.let { fields ->
+ textData.standoffFields = fields.toMutableList()
+ }
+ }
}
private fun enqueueKrillCompression(textId: String, textData: KrillJsonGenerator.KrillTextData) {
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/StandoffMetadata.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/StandoffMetadata.kt
new file mode 100644
index 0000000..4846602
--- /dev/null
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/StandoffMetadata.kt
@@ -0,0 +1,159 @@
+package de.ids_mannheim.korapxmltools
+
+import de.ids_mannheim.korapxmltools.formatters.KrillJsonGenerator.StandoffField
+import org.w3c.dom.Element
+import java.io.File
+import java.util.logging.Logger
+import javax.xml.parsers.DocumentBuilderFactory
+
+/**
+ * Reads stand-off metadata XML files (one file per corpus, in the `<standOff>`
+ * form produced by e.g. the wiki-taxonomy classifier) and turns them into
+ * ready-to-emit Krill metadata fields keyed by the text's docid.
+ *
+ * The format is TEI-namespaced and reuses TEI's classification vocabulary
+ * (taxonomy/category/catRef), but is not conformant TEI P5: `standOff`,
+ * `metadataLayer`, `textRef` and `source` are local extensions.
+ *
+ * Layout:
+ *
+ * <standOff>
+ * <metadataLayer xml:id="wikidomain" type="classification">
+ * <taxonomy .../> (declaration, ignored here)
+ * <textRef target="REI_RBR.00473"> (target == raw_text/@docid)
+ * <catRef target="#Language" n="1" cert="0.39"/>
+ * </textRef>
+ * </metadataLayer>
+ * </standOff>
+ *
+ * Selection policy for classification layers: keep the highest-ranked categories
+ * whose `cert` is at least [classificationMinCert], capped at [classificationTopN],
+ * and emit them as a single `type:keywords` field named after the layer's id.
+ *
+ * By default we ingest *everything* present in the file: the annotation tool that
+ * produced it has already applied its own top-k/threshold, and we want to be able
+ * to re-use existing classifications as-is without redoing them. The threshold
+ * knobs below are kept so a future CLI option can re-enable curation at index time
+ * (e.g. top 2 with cert >= 0.3).
+ */
+object StandoffMetadata {
+ private val LOGGER = Logger.getLogger(StandoffMetadata::class.java.name)
+
+ /** Default indexing selection: take all categories present in the file. */
+ const val DEFAULT_TOP_N = Int.MAX_VALUE
+ const val DEFAULT_MIN_CERT = 0.0
+
+ private val factory: DocumentBuilderFactory by lazy {
+ DocumentBuilderFactory.newInstance().apply {
+ isNamespaceAware = true
+ }
+ }
+
+ /**
+ * Cheap recogniser used to auto-detect stand-off metadata among the input
+ * files (so no dedicated CLI option is needed). True when [path] is an `.xml`
+ * file whose root element is `standOff`.
+ */
+ fun isStandoffMetadataFile(path: String): Boolean {
+ if (!path.endsWith(".xml", ignoreCase = true)) return false
+ val file = File(path)
+ if (!file.isFile) return false
+ return try {
+ val root = factory.newDocumentBuilder().parse(file).documentElement
+ root.localName == "standOff" || root.tagName == "standOff"
+ } catch (e: Exception) {
+ LOGGER.fine("Not a stand-off metadata file ($path): ${e.message}")
+ false
+ }
+ }
+
+ /**
+ * Parse [path] and merge its per-text fields into [target] (keyed by docid).
+ */
+ fun parseInto(
+ path: String,
+ target: MutableMap<String, MutableList<StandoffField>>,
+ classificationTopN: Int = DEFAULT_TOP_N,
+ classificationMinCert: Double = DEFAULT_MIN_CERT
+ ) {
+ val root = factory.newDocumentBuilder().parse(File(path)).documentElement
+ var layers = 0
+ var entries = 0
+ root.childElements("metadataLayer").forEach { layer ->
+ layers++
+ val key = layer.idOrKey()
+ if (key.isNullOrBlank()) {
+ LOGGER.warning("Stand-off metadata layer without xml:id/key in $path; skipping")
+ return@forEach
+ }
+ val type = layer.getAttribute("type").ifBlank { "classification" }
+ layer.childElements("textRef").forEach { textRef ->
+ val docId = textRef.getAttribute("target").removePrefix("#")
+ if (docId.isBlank()) return@forEach
+ val field = when (type) {
+ "classification" -> classificationField(key, textRef, classificationTopN, classificationMinCert)
+ "links" -> linkField(key, textRef)
+ else -> {
+ LOGGER.warning("Unknown stand-off metadata layer type '$type' in $path; skipping")
+ null
+ }
+ } ?: return@forEach
+ target.getOrPut(docId) { mutableListOf() }.add(field)
+ entries++
+ }
+ }
+ LOGGER.info("Loaded stand-off metadata from $path: $layers layer(s), $entries text entr(ies)")
+ }
+
+ private fun classificationField(
+ key: String,
+ textRef: Element,
+ topN: Int,
+ minCert: Double
+ ): StandoffField? {
+ val cats = textRef.childElements("catRef")
+ .mapNotNull { catRef ->
+ val target = catRef.getAttribute("target").removePrefix("#")
+ if (target.isBlank()) return@mapNotNull null
+ val cert = catRef.getAttribute("cert").toDoubleOrNull() ?: Double.NaN
+ val rank = catRef.getAttribute("n").toIntOrNull() ?: Int.MAX_VALUE
+ Triple(target, cert, rank)
+ }
+ .filter { it.second.isNaN() || it.second >= minCert }
+ .sortedBy { it.third }
+ .take(topN)
+ .map { it.first }
+ return if (cats.isEmpty()) null else StandoffField(key, "type:keywords", cats)
+ }
+
+ private fun linkField(key: String, textRef: Element): StandoffField? {
+ // Emit the first <ref> as a KorAP link attachment, matching the encoding
+ // used for textExternalLink in KrillJsonGenerator.
+ val ref = textRef.childElements("ref").firstOrNull() ?: return null
+ val url = ref.getAttribute("target")
+ if (url.isBlank()) return null
+ val title = ref.textContent?.trim()?.ifBlank { null } ?: "Link"
+ val encodedUrl = url.replace(":", "%3A").replace("/", "%2F")
+ return StandoffField(key, "type:attachement", "data:application/x.korap-link;title=$title,$encodedUrl")
+ }
+
+ private fun Element.idOrKey(): String? {
+ val id = getAttributeNS("http://www.w3.org/XML/1998/namespace", "id")
+ if (id.isNotBlank()) return id
+ val xmlId = getAttribute("xml:id")
+ if (xmlId.isNotBlank()) return xmlId
+ return getAttribute("key").ifBlank { null }
+ }
+
+ private fun Element.childElements(localName: String): List<Element> {
+ val result = mutableListOf<Element>()
+ val children = childNodes
+ for (i in 0 until children.length) {
+ val node = children.item(i)
+ if (node is Element && (node.localName == localName || node.tagName == localName)) {
+ result.add(node)
+ }
+ }
+ return result
+ }
+}
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
index b57af66..e3fafea 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
@@ -47,7 +47,23 @@
var extractedAttributes: MutableMap<String, String> = mutableMapOf(),
var lpSentencesCollected: Boolean = false,
var sentencesCollectedByFoundry: MutableSet<String> = mutableSetOf(),
- var constituencyCollectedByFoundry: MutableSet<String> = mutableSetOf()
+ var constituencyCollectedByFoundry: MutableSet<String> = mutableSetOf(),
+ // Extra metadata fields derived from stand-off metadata files, already
+ // resolved to their final Krill type/value (see StandoffMetadata).
+ var standoffFields: MutableList<StandoffField> = mutableListOf()
+ )
+
+ /**
+ * A ready-to-emit Krill metadata field coming from a stand-off metadata file.
+ *
+ * @param key the Krill field name (koral:field "key")
+ * @param type the Krill field type, e.g. "type:keywords" or "type:attachement"
+ * @param value either a List<String> (keywords) or a String (everything else)
+ */
+ data class StandoffField(
+ val key: String,
+ val type: String,
+ val value: Any
)
/**
@@ -233,6 +249,25 @@
)))
}
+ // Stand-off metadata fields (already typed/selected upstream). Skip any key
+ // that a header field above already emitted, so stand-off never clobbers it.
+ val emittedKeys = (if (sigleParts.size >= 3)
+ mutableSetOf("corpusSigle", "docSigle", "textSigle") else mutableSetOf())
+ emittedKeys.addAll(resolvedHeaderMetadata.keys)
+ textData.standoffFields.forEach { field ->
+ if (!emittedKeys.add(field.key)) return@forEach
+ val fieldValue = when (val v = field.value) {
+ is List<*> -> jsonArray(v.map { jsonString(it.toString()) })
+ else -> jsonString(v.toString())
+ }
+ fields.add(jsonObject(listOf(
+ "key" to jsonString(field.key),
+ "@type" to jsonString("koral:field"),
+ "value" to fieldValue,
+ "type" to jsonString(field.type)
+ )))
+ }
+
sb.append(fields.joinToString(","))
sb.append("],")
diff --git a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
index fb742fb..431e31b 100644
--- a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
+++ b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
@@ -1224,4 +1224,33 @@
}
}
}
+
+ @Test
+ fun standoffMetadataAddsClassificationKeywords() {
+ val baseZip = loadResource("rei_sample.zip").path
+ val standoff = loadResource("rei_sample.domains.meta.xml").path
+
+ val generatedTar = ensureKrillTar("rei_standoff", "rei_sample.krill.tar") { outputDir ->
+ arrayOf("-t", "krill", "-q", "-D", outputDir.path, baseZip, standoff)
+ }
+ assertTrue(generatedTar.exists())
+
+ val jsonByFile = readKrillJson(generatedTar)
+ val rei473 = jsonByFile.entries.first { it.key.startsWith("REI-RBR-00473") }.value
+
+ // The layer xml:id ("wikidomain") becomes the Krill field key, typed as keywords.
+ val field = Regex(
+ """"key":"wikidomain","@type":"koral:field","value":\[([^\]]*)\],"type":"type:keywords""""
+ ).find(rei473)
+ assertNotNull(field, "Expected a wikidomain keywords field in REI_RBR.00473")
+
+ // Default policy ingests everything in the file, ordered by rank (no curation
+ // at index time). REI_RBR.00473 has all five categories.
+ val values = field.groupValues[1]
+ assertEquals(
+ """"Language","History","Culture","Mass_media","People"""",
+ values,
+ "All categories present in the file should be indexed, in rank order"
+ )
+ }
}
diff --git a/app/src/test/resources/rei_sample.domains.meta.xml b/app/src/test/resources/rei_sample.domains.meta.xml
new file mode 100644
index 0000000..4138bd1
--- /dev/null
+++ b/app/src/test/resources/rei_sample.domains.meta.xml
@@ -0,0 +1,107 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Stand-off metadata for the corpus contained in rei_sample.zip.
+
+ In the TEI namespace and reusing TEI's classification vocabulary
+ (taxonomy/category/catDesc/catRef), but NOT conformant TEI P5: standOff,
+ metadataLayer, textRef and source are local KorAP extensions.
+
+ Conventions:
+ * One <metadataLayer> per logical metadata category ("foundry"); its
+ xml:id is the Krill metadata field key (here: "wikidomain"). The taxonomy
+ has its own distinct xml:id ("wikitaxonomy") so the two never collide.
+ * <textRef target="..."> joins on raw_text/@docid from data.xml
+ (e.g. REI_RBR.00473) - NOT on the slash-form textSigle. No '#':
+ the docid is an external corpus identifier, not a fragment of this file.
+ * Intra-document pointers (catRef/@scheme, catRef/@target) DO use '#'
+ because they reference xml:ids declared within this file.
+ * catRef/@n = 1-based rank; catRef/@cert = classifier confidence.
+-->
+<standOff xmlns="http://www.tei-c.org/ns/1.0" version="1.0">
+
+ <!-- ===================================================================
+ Layer 1: Wikipedia top-level topic-domain classification
+ =================================================================== -->
+ <metadataLayer xml:id="wikidomain" type="classification" join="docid">
+
+ <!-- Provenance: this layer is machine-derived, so describe how. -->
+ <source tool="wiki-taxonomy" model="epoch_33-step_183090"
+ date="2026-06-06" select="top5" threshold="0.05" scoreType="confidence"/>
+
+ <taxonomy xml:id="wikitaxonomy">
+ <desc>Wikipedia top-level topic domains (flat, single level)</desc>
+ <category xml:id="Academic_disciplines"><catDesc>Academic disciplines</catDesc></category>
+ <category xml:id="Business"><catDesc>Business</catDesc></category>
+ <category xml:id="Communication"><catDesc>Communication</catDesc></category>
+ <category xml:id="Concepts"><catDesc>Concepts</catDesc></category>
+ <category xml:id="Culture"><catDesc>Culture</catDesc></category>
+ <category xml:id="Economy"><catDesc>Economy</catDesc></category>
+ <category xml:id="Education"><catDesc>Education</catDesc></category>
+ <category xml:id="Energy"><catDesc>Energy</catDesc></category>
+ <category xml:id="Engineering"><catDesc>Engineering</catDesc></category>
+ <category xml:id="Entertainment"><catDesc>Entertainment</catDesc></category>
+ <category xml:id="Entities"><catDesc>Entities</catDesc></category>
+ <category xml:id="Food_drink"><catDesc>Food drink</catDesc></category>
+ <category xml:id="Geography"><catDesc>Geography</catDesc></category>
+ <category xml:id="Government"><catDesc>Government</catDesc></category>
+ <category xml:id="Health"><catDesc>Health</catDesc></category>
+ <category xml:id="History"><catDesc>History</catDesc></category>
+ <category xml:id="Human_behavior"><catDesc>Human behavior</catDesc></category>
+ <category xml:id="Humanities"><catDesc>Humanities</catDesc></category>
+ <category xml:id="Information"><catDesc>Information</catDesc></category>
+ <category xml:id="Internet"><catDesc>Internet</catDesc></category>
+ <category xml:id="Knowledge"><catDesc>Knowledge</catDesc></category>
+ <category xml:id="Language"><catDesc>Language</catDesc></category>
+ <category xml:id="Law"><catDesc>Law</catDesc></category>
+ <category xml:id="Life"><catDesc>Life</catDesc></category>
+ <category xml:id="Lists"><catDesc>Lists</catDesc></category>
+ <category xml:id="Mass_media"><catDesc>Mass media</catDesc></category>
+ <category xml:id="Mathematics"><catDesc>Mathematics</catDesc></category>
+ <category xml:id="Military"><catDesc>Military</catDesc></category>
+ <category xml:id="Nature"><catDesc>Nature</catDesc></category>
+ <category xml:id="People"><catDesc>People</catDesc></category>
+ <category xml:id="Philosophy"><catDesc>Philosophy</catDesc></category>
+ <category xml:id="Politics"><catDesc>Politics</catDesc></category>
+ <category xml:id="Religion"><catDesc>Religion</catDesc></category>
+ <category xml:id="Science"><catDesc>Science</catDesc></category>
+ <category xml:id="Society"><catDesc>Society</catDesc></category>
+ <category xml:id="Sports"><catDesc>Sports</catDesc></category>
+ <category xml:id="Technology"><catDesc>Technology</catDesc></category>
+ <category xml:id="Time"><catDesc>Time</catDesc></category>
+ <category xml:id="Universe"><catDesc>Universe</catDesc></category>
+ </taxonomy>
+
+ <textRef target="REI_RBR.00473">
+ <catRef scheme="#wikitaxonomy" target="#Language" n="1" cert="0.3947"/>
+ <catRef scheme="#wikitaxonomy" target="#History" n="2" cert="0.3142"/>
+ <catRef scheme="#wikitaxonomy" target="#Culture" n="3" cert="0.2739"/>
+ <catRef scheme="#wikitaxonomy" target="#Mass_media" n="4" cert="0.2524"/>
+ <catRef scheme="#wikitaxonomy" target="#People" n="5" cert="0.2486"/>
+ </textRef>
+ <textRef target="REI_RBR.01123">
+ <catRef scheme="#wikitaxonomy" target="#Lists" n="1" cert="0.4853"/>
+ <catRef scheme="#wikitaxonomy" target="#History" n="2" cert="0.4571"/>
+ <catRef scheme="#wikitaxonomy" target="#Politics" n="3" cert="0.3770"/>
+ <catRef scheme="#wikitaxonomy" target="#Culture" n="4" cert="0.3549"/>
+ <catRef scheme="#wikitaxonomy" target="#Language" n="5" cert="0.3525"/>
+ </textRef>
+ <textRef target="REI_RBR.01561">
+ <catRef scheme="#wikitaxonomy" target="#Language" n="1" cert="0.6705"/>
+ <catRef scheme="#wikitaxonomy" target="#Lists" n="2" cert="0.5889"/>
+ <catRef scheme="#wikitaxonomy" target="#History" n="3" cert="0.5108"/>
+ <catRef scheme="#wikitaxonomy" target="#Culture" n="4" cert="0.4956"/>
+ <catRef scheme="#wikitaxonomy" target="#Politics" n="5" cert="0.3944"/>
+ </textRef>
+ </metadataLayer>
+
+ <!-- ===================================================================
+ Layer 2: external links (illustrative - same container, different payload)
+ =================================================================== -->
+ <metadataLayer xml:id="external-links" type="links" join="docid">
+ <source tool="manual" date="2026-06-06"/>
+ <textRef target="REI_RBR.00473">
+ <ref type="page_url" target="https://politische-reden.eu/#data"/>
+ </textRef>
+ </metadataLayer>
+
+</standOff>