Pass I5 xenoData metadata blocks to Krill output (issue #54)
I5 <xenoData> blocks carry corpus-provided metadata (e.g. the RPK
denomination) that previously never reached Krill, so it was neither
displayed nor searchable. Mirror KorAP-XML-Krill's KorAP::XML::Meta::I5
handling (commit 24ad3c0): parse each <meta> into a typed Krill field,
mapping the meta type to the field type (string/keyword/text/date ->
type:string/keywords/text/date; attachment/uri -> type:attachement) and
deriving the key as <project>.<name>. For doc/corpus headers the name is
level-prefixed (docRcpnt/corpusRcpnt) so the same meta stays distinct per
level; repeated keywords accumulate, repeated scalars keep the last value.
Fields are collected per level and inherited corpus -> doc -> text, then
attached to each text's Krill stand-off fields (Krill-only by construction).
Tests: parseXenoData unit coverage for the type/key mapping and edge cases
(text/date metas, empty/unknown-type skipping, doc/corpus prefixing), plus
a full-pipeline test over a new ked_sample.zip fixture (packaged from
KorAP-XML-Krill t/real/corpus/KED) that mirrors the Perl ked.t assertions.
Resolves #54
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Change-Id: I2d895bbc05cfe203a6e47ad4932c9ea1eb6fd0a2
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index c0e9074..739f8c7 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -1314,6 +1314,11 @@
val readyKrillTextIds: java.util.concurrent.BlockingQueue<String> = java.util.concurrent.LinkedBlockingQueue()
val corpusMetadata: ConcurrentHashMap<String, MutableMap<String, Any>> = ConcurrentHashMap()
val docMetadata: ConcurrentHashMap<String, MutableMap<String, Any>> = ConcurrentHashMap()
+ // I5 <xenoData> fields parsed from headers, kept per level so they inherit
+ // corpus -> doc -> text and are attached to each text's Krill stand-off fields.
+ val corpusXenoData: ConcurrentHashMap<String, MutableList<KrillJsonGenerator.StandoffField>> = ConcurrentHashMap()
+ val docXenoData: ConcurrentHashMap<String, MutableList<KrillJsonGenerator.StandoffField>> = ConcurrentHashMap()
+ val textXenoData: ConcurrentHashMap<String, MutableList<KrillJsonGenerator.StandoffField>> = ConcurrentHashMap()
val expectedFoundries: MutableSet<String> = mutableSetOf("base")
val processedFoundries: MutableSet<String> = mutableSetOf()
var krillOutputCount = java.util.concurrent.atomic.AtomicInteger(0)
@@ -1398,6 +1403,9 @@
krillCompressionStartNanos.clear()
corpusMetadata.clear()
docMetadata.clear()
+ corpusXenoData.clear()
+ docXenoData.clear()
+ textXenoData.clear()
zipInventory.clear()
processedTextsPerZip.clear()
outputTexts.clear()
@@ -5565,6 +5573,9 @@
mergeExtractedKrillMetadata(textData.headerMetadata, extractKrillHeaderMetadata(headerRoot))
LOGGER.fine("Collected ${textData.headerMetadata.size} metadata fields for $docId")
}
+ StandoffMetadata.parseXenoData(headerRoot, "text").takeIf { it.isNotEmpty() }?.let {
+ textXenoData[docId] = it.toMutableList()
+ }
}
private fun extractKrillHeaderMetadata(headerRoot: Element): MutableMap<String, Any> {
@@ -5798,6 +5809,9 @@
)
LOGGER.fine("Collected ${metadata.size} corpus-level metadata fields for $corpusSigle")
}
+ StandoffMetadata.parseXenoData(headerRoot, "corpus").takeIf { it.isNotEmpty() }?.let {
+ corpusXenoData[corpusSigle] = it.toMutableList()
+ }
}
// Collect document-level metadata from doc header
@@ -5810,6 +5824,9 @@
metadata.putIfNotBlank("docAuthor", headerRoot.firstText("h.author") ?: metadata["author"] as? String)
LOGGER.fine("Collected ${metadata.size} doc-level metadata fields for $docSigle")
}
+ StandoffMetadata.parseXenoData(headerRoot, "doc").takeIf { it.isNotEmpty() }?.let {
+ docXenoData[docSigle] = it.toMutableList()
+ }
}
// Collect base text data (text, tokens, sentences) for krill format
@@ -6213,11 +6230,21 @@
textData.headerMetadata.clear()
textData.headerMetadata.putAll(resolvedMetadata)
- // Attach any stand-off metadata fields for this text (joined by docid).
- if (standoffMetadata.isNotEmpty()) {
- standoffMetadata[textId]?.let { fields ->
- textData.standoffFields = fields.toMutableList()
- }
+ // Attach stand-off and xenodata metadata fields for this text. Xenodata
+ // inherits corpus -> doc -> text; the per-level key prefixes (corpus*/doc*)
+ // keep the levels distinct so they never clobber each other.
+ val textIdWithSlashes = textId.replace("_", "/").replace(".", "/")
+ val sigleParts = textIdWithSlashes.split("/")
+ val corpusSigle = sigleParts.firstOrNull().orEmpty()
+ val docSigle = sigleParts.take(2).joinToString("/")
+
+ val combinedStandoff = mutableListOf<KrillJsonGenerator.StandoffField>()
+ standoffMetadata[textId]?.let { combinedStandoff.addAll(it) }
+ corpusXenoData[corpusSigle]?.let { combinedStandoff.addAll(it) }
+ docXenoData[docSigle]?.let { combinedStandoff.addAll(it) }
+ textXenoData[textId]?.let { combinedStandoff.addAll(it) }
+ if (combinedStandoff.isNotEmpty()) {
+ textData.standoffFields = combinedStandoff
}
}
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/StandoffMetadata.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/StandoffMetadata.kt
index cd86b76..9ff2393 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/StandoffMetadata.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/StandoffMetadata.kt
@@ -156,4 +156,136 @@
}
return result
}
+
+ /**
+ * Parse the I5 `<xenoData>` block of a header element into ready-to-emit Krill
+ * fields, mirroring KorAP-XML-Krill's `KorAP::XML::Meta::I5` xenodata handling.
+ *
+ * Each `<meta name="..." type="..." [project="..."] [desc="..."]>VALUE</meta>`
+ * becomes one Krill field. The meta `type` selects the field type:
+ * string -> type:string, keyword -> type:keywords, text -> type:text,
+ * date -> type:date, attachment -> type:attachement, uri -> type:attachement
+ * (value wrapped as a korap-link data URI). Unknown types are skipped.
+ *
+ * The field key is `<project>.<name>` (the `<project>.` part only when a
+ * project attribute is present). For doc/corpus headers the meta name is
+ * prefixed with the level ("doc"/"corpus") and upper-cased, so the same meta
+ * name stays a distinct field across levels (text `rcpnt`, doc `docRcpnt`,
+ * corpus `corpusRcpnt`). Repeated keyword metas with the same key accumulate
+ * into a list; repeated scalars keep the last value.
+ *
+ * @param header the idsHeader root element (may be null)
+ * @param headerType "text", "doc" or "corpus"
+ */
+ fun parseXenoData(header: Element?, headerType: String): List<StandoffField> {
+ if (header == null) return emptyList()
+ val xeno = header.getElementsByTagName("xenoData")
+ .let { if (it.length == 0) null else it.item(0) as? Element }
+ ?: return emptyList()
+
+ // Keyed by the internal (type-prefixed) key so metas only collide when they
+ // would in Perl, too; LinkedHashMap preserves document order of first sight.
+ val ordered = LinkedHashMap<String, XenoEntry>()
+ val metas = xeno.getElementsByTagName("meta")
+ for (i in 0 until metas.length) {
+ val meta = metas.item(i) as? Element ?: continue
+ val name = meta.getAttribute("name").trim()
+ if (name.isEmpty()) continue
+ val value = squish(meta.textContent ?: "")
+ if (value.isEmpty()) continue
+ val xtype = meta.getAttribute("type").trim()
+ if (xtype.isEmpty()) continue
+
+ val typePrefix = when (xtype) {
+ "string" -> "S_"
+ "keyword" -> "K_"
+ "text" -> "T_"
+ "date" -> "D_"
+ "attachment", "uri" -> "A_"
+ else -> {
+ LOGGER.warning("Unknown xenodata type: $xtype")
+ continue
+ }
+ }
+ val fieldType = when (xtype) {
+ "string" -> "type:string"
+ "keyword" -> "type:keywords"
+ "text" -> "type:text"
+ "date" -> "type:date"
+ else -> "type:attachement" // attachment, uri
+ }
+ val fieldValue = when (xtype) {
+ "uri" -> korapDataUri(value, meta.getAttribute("desc").trim().ifBlank { value })
+ "attachment" -> if (value.startsWith("data:")) value else "data:,$value"
+ "date" -> normalizeXenoDate(value)
+ else -> value
+ }
+
+ val project = meta.getAttribute("project").trim()
+ val nameKey = if (headerType == "doc" || headerType == "corpus") {
+ headerType + name.replaceFirstChar { it.uppercase() }
+ } else {
+ name
+ }
+ val internalKey = typePrefix + (if (project.isNotEmpty()) "$project." else "") + nameKey
+
+ val entry = ordered.getOrPut(internalKey) { XenoEntry(fieldKey(internalKey), fieldType) }
+ if (xtype == "keyword") {
+ entry.keywords.add(fieldValue)
+ } else {
+ entry.scalar = fieldValue
+ }
+ }
+
+ return ordered.values.map { entry ->
+ if (entry.type == "type:keywords") {
+ StandoffField(entry.key, entry.type, entry.keywords.toList())
+ } else {
+ StandoffField(entry.key, entry.type, entry.scalar ?: "")
+ }
+ }
+ }
+
+ private class XenoEntry(val key: String, val type: String) {
+ val keywords = mutableListOf<String>()
+ var scalar: String? = null
+ }
+
+ /**
+ * Derive the Krill field key from the internal type-prefixed key, mirroring
+ * KorAP::XML::Meta::Base::_k: strip the 2-char type prefix, camel-case `_x`
+ * sequences and upper-case a trailing "id".
+ */
+ private fun fieldKey(internalKey: String): String {
+ var x = internalKey.substring(2)
+ x = Regex("_(\\w)").replace(x) { it.groupValues[1].uppercase() }
+ x = x.replace(Regex("(?i)id$"), "ID")
+ return x
+ }
+
+ /** Collapse internal whitespace runs and trim; an all-dashes value becomes empty. */
+ private fun squish(s: String): String {
+ var v = s.replace(Regex("\\s\\s+"), " ").trim()
+ if (v.matches(Regex("^-+$"))) v = ""
+ return v
+ }
+
+ /** Normalise an 8-digit YYYYMMDD date to YYYY[-MM[-DD]]; pass other forms through. */
+ private fun normalizeXenoDate(value: String): String {
+ val m = Regex("^(\\d{4})(\\d{2})(\\d{2})$").find(value) ?: return value
+ val (y, mo, d) = m.destructured
+ val sb = StringBuilder(y)
+ if (mo != "00") {
+ sb.append("-").append(mo)
+ if (d != "00") sb.append("-").append(d)
+ }
+ return sb.toString()
+ }
+
+ /** Build a `data:application/x.korap-link` URI, mirroring Meta::Base::korap_data_uri. */
+ private fun korapDataUri(data: String, title: String): String =
+ "data:application/x.korap-link;title=${urlEscape(title)},${urlEscape(data)}"
+
+ private fun urlEscape(s: String): String =
+ java.net.URLEncoder.encode(s, "UTF-8").replace("+", "%20")
}
diff --git a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
index 89f229b..aec8a92 100644
--- a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
+++ b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
@@ -14,6 +14,7 @@
import java.util.zip.GZIPInputStream
import javax.xml.parsers.DocumentBuilderFactory
import kotlin.test.Test
+import kotlin.test.assertContains
import kotlin.test.assertEquals
import kotlin.test.assertTrue
import kotlin.test.assertFalse
@@ -1285,6 +1286,152 @@
)
}
+ /**
+ * Regression tests for https://github.com/KorAP/korapxmltool/issues/54
+ *
+ * I5 `<xenoData>` blocks carry corpus-provided metadata (e.g. the RPK
+ * denomination) that must reach Krill so it is displayed and searchable.
+ * Mirrors KorAP-XML-Krill commit 24ad3c0 (KorAP::XML::Meta::I5) and its
+ * t/real/ked.t expectations.
+ */
+ @Test
+ fun parseXenoDataMapsTypesKeysAndAccumulatesKeywords() {
+ // Text-level xenoData from KED/KLX/03212 (KorAP-XML-Krill t/real), plus the
+ // project-less text/date metas from the issue's RPK example.
+ val fields = StandoffMetadata.parseXenoData(
+ headerElement(
+ """
+ <idsHeader>
+ <xenoData>
+ <meta name="rcpnt" project="KED" type="keyword" desc="recipient group">kinder</meta>
+ <meta name="rcpntLabel" project="KED" type="attachment" desc="recipient group capitalized">Kinder</meta>
+ <meta name="strtgy" project="KED" type="keyword" desc="strategy">erklaeren</meta>
+ <meta name="cover1Herder" project="KED" type="string" desc="text coverage 1k">0.58</meta>
+ <meta name="cover5Herder" project="KED" type="string">0.66</meta>
+ <meta name="topicLabel" project="KED" type="attachment">Gesundheit und Krankheit</meta>
+ <!-- repeated metas: keyword accumulates, scalar keeps the last -->
+ <meta name="strtgy" project="KED" type="keyword">beschreiben</meta>
+ <meta name="cover5Herder" project="KED" type="string">0.67</meta>
+ <meta name="GUID" type="text">37f38083-948d-403c-a2c2-8631e8e4a91d</meta>
+ <meta name="Datum" type="date">2022-01-27</meta>
+ <meta name="empty" type="string"> </meta>
+ <meta name="bogus" type="frobnicate">x</meta>
+ </xenoData>
+ </idsHeader>
+ """
+ ),
+ "text"
+ ).associateBy { it.key }
+
+ // attachment -> type:attachement, value wrapped as a data: URI
+ assertEquals("type:attachement", fields.getValue("KED.rcpntLabel").type)
+ assertEquals("data:,Kinder", fields.getValue("KED.rcpntLabel").value)
+ assertEquals("data:,Gesundheit und Krankheit", fields.getValue("KED.topicLabel").value)
+
+ // string -> type:string
+ assertEquals("type:string", fields.getValue("KED.cover1Herder").type)
+ assertEquals("0.58", fields.getValue("KED.cover1Herder").value)
+ // repeated scalar keeps the last value
+ assertEquals("0.67", fields.getValue("KED.cover5Herder").value)
+
+ // keyword -> type:keywords; repeats accumulate in document order; singles stay lists
+ assertEquals("type:keywords", fields.getValue("KED.strtgy").type)
+ assertEquals(listOf("erklaeren", "beschreiben"), fields.getValue("KED.strtgy").value)
+ assertEquals(listOf("kinder"), fields.getValue("KED.rcpnt").value)
+
+ // text -> type:text, project-less key (issue's RPK example)
+ assertEquals("type:text", fields.getValue("GUID").type)
+ assertEquals("37f38083-948d-403c-a2c2-8631e8e4a91d", fields.getValue("GUID").value)
+ // date -> type:date, ISO value passes through
+ assertEquals("type:date", fields.getValue("Datum").type)
+ assertEquals("2022-01-27", fields.getValue("Datum").value)
+
+ // Empty values and unknown types are skipped
+ assertFalse(fields.containsKey("empty"))
+ assertFalse(fields.containsKey("bogus"))
+ }
+
+ @Test
+ fun parseXenoDataPrefixesNamesForDocAndCorpusLevels() {
+ val xml =
+ """
+ <idsHeader>
+ <xenoData>
+ <meta name="rcpnt" project="KED" type="keyword" desc="recipient group">kinder</meta>
+ <meta name="rcpntLabel" project="KED" type="attachment">Kinder</meta>
+ </xenoData>
+ </idsHeader>
+ """
+ val docFields = StandoffMetadata.parseXenoData(headerElement(xml), "doc").associateBy { it.key }
+ assertEquals(listOf("kinder"), docFields.getValue("KED.docRcpnt").value)
+ assertEquals("data:,Kinder", docFields.getValue("KED.docRcpntLabel").value)
+
+ val corpusFields = StandoffMetadata.parseXenoData(headerElement(xml), "corpus").associateBy { it.key }
+ assertEquals(listOf("kinder"), corpusFields.getValue("KED.corpusRcpnt").value)
+ assertEquals("data:,Kinder", corpusFields.getValue("KED.corpusRcpntLabel").value)
+ }
+
+ /**
+ * Full-pipeline mirror of KorAP-XML-Krill's t/real/ked.t (commit 24ad3c0):
+ * the KED/KLX/03212 fixture carries xenoData at text, doc and corpus level,
+ * and its Krill JSON must expose every meta as a typed field. The Perl test
+ * asserts on the internal `<typeprefix>_<key>` meta keys (e.g. A_KED.topicLabel);
+ * here the type prefix has become the koral:field `type` and the key is the
+ * remainder (KED.topicLabel), so the assertions correspond one-to-one.
+ */
+ @Test
+ fun xenoDataFromKedCorpusMatchesPerlReference() {
+ val tar = ensureKrillTar("ked_xenodata", "ked_sample.krill.tar") { outputDir ->
+ arrayOf("-t", "krill", "-q", "-D", outputDir.path, loadResource("ked_sample.zip").path)
+ }
+ val json = readKrillJson(tar).getValue("KED-KLX-03212.json")
+
+ fun field(key: String, value: String, type: String) =
+ """"key":"$key","@type":"koral:field","value":"$value","type":"$type""""
+ fun keywords(key: String, vararg values: String) =
+ """"key":"$key","@type":"koral:field","value":[""" +
+ values.joinToString(",") { "\"$it\"" } + """],"type":"type:keywords""""
+
+ // attachment metas -> type:attachement, value wrapped as a data: URI (A_ in ked.t)
+ assertContains(json, field("KED.topicLabel", "data:,Gesundheit und Krankheit", "type:attachement"))
+ assertContains(json, field("KED.strtgyLabel", "data:,Erklären", "type:attachement"))
+ assertContains(json, field("KED.txttypLabel", "data:,Lexikonartikel", "type:attachement"))
+ assertContains(json, field("KED.rcpntLabel", "data:,Kinder", "type:attachement"))
+ assertContains(json, field("KED.nToks", "data:,308", "type:attachement"))
+ assertContains(json, field("KED.nSent", "data:,28", "type:attachement"))
+ assertContains(json, field("KED.nTyps", "data:,188", "type:attachement"))
+ assertContains(json, field("KED.nToksSentMd", "data:,11.0", "type:attachement"))
+ assertContains(json, field("KED.nPunct1kTks", "data:,129.87", "type:attachement"))
+
+ // string metas -> type:string (S_ in ked.t)
+ assertContains(json, field("KED.cover1Herder", "0.58", "type:string"))
+ assertContains(json, field("KED.cover2Herder", "0.61", "type:string"))
+ assertContains(json, field("KED.cover3Herder", "0.62", "type:string"))
+ assertContains(json, field("KED.cover4Herder", "0.65", "type:string"))
+ assertContains(json, field("KED.nPara", "5", "type:string"))
+ // Repeated scalar keeps the last value (0.66 then "added for testing" 0.67)
+ assertContains(json, field("KED.cover5Herder", "0.67", "type:string"))
+
+ // keyword metas -> type:keywords (K_ in ked.t); repeats accumulate in order
+ assertContains(json, keywords("KED.strtgy", "erklaeren", "beschreiben"))
+ assertContains(json, keywords("KED.topic", "gesundheit_krankheit"))
+ assertContains(json, keywords("KED.txttyp", "lexikonartikel"))
+ assertContains(json, keywords("KED.rcpnt", "kinder"))
+
+ // doc- and corpus-level xenoData inherit onto the text under level-prefixed
+ // keys, so they never collide with the text-level rcpnt field.
+ assertContains(json, keywords("KED.docRcpnt", "kinder"))
+ assertContains(json, field("KED.docRcpntLabel", "data:,Kinder", "type:attachement"))
+ assertContains(json, keywords("KED.corpusRcpnt", "kinder"))
+ assertContains(json, field("KED.corpusRcpntLabel", "data:,Kinder", "type:attachement"))
+
+ // Anchor: the sigle and a standard header field are still present. (Sigle
+ // fields use a different property order, so check them order-independently.)
+ assertContains(json, "\"key\":\"textSigle\"")
+ assertContains(json, "\"value\":\"KED/KLX/03212\"")
+ assertContains(json, field("title", "Flöhe", "type:text"))
+ }
+
@Test
fun correctedMetadataFieldNamesByDefault() {
val baseZip = loadResource("rei_sample.zip").path
diff --git a/app/src/test/resources/ked_sample.zip b/app/src/test/resources/ked_sample.zip
new file mode 100644
index 0000000..bd8f783
--- /dev/null
+++ b/app/src/test/resources/ked_sample.zip
Binary files differ