Support corpora with custom tokenization and annotations
Corpora converted from TEI with their own tokenization and POS/lemma
annotations ship them inside the base ZIP in a custom foundry folder
(e.g. cmc/morpho.xml) and have no base/tokens.xml. Two fixes for this:
- Derive the foundry of annotation entries in base ZIPs from the folder
name instead of the ZIP file name, so Krill output indexes the
annotations (cmc/p, cmc/l, ...) instead of dropping them as "base",
and CoNLL-U reports the correct "# foundry =" comment.
- Register token spans from morpho.xml in the Krill text data when no
base/tokens.xml provides them, with tokenSource set to e.g.
"cmc#morpho"; previously the Krill token stream was empty for such
corpora.
Additionally, when structure.xml has no sentence (s) spans, sentence
segmentation now falls back to other segment-like TEI elements
(posting, l, seg, u, in this order), so integrated parsers and KorAP
sentence queries work on such corpora.
Tested with a new dck_sample.zip resource, a two-text excerpt from the
CC BY licensed Dortmunder Chat-Korpus; in one text the s spans are
removed to exercise the sentence fallback.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Change-Id: Iacf97fb151edd8c8d2add244e71c3d32282531f1
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 46e238d..8177fef 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,15 @@
## [Unreleased]
+### Added
+
+- Sentence segmentation fallback: when a corpus has no `s` spans in `structure.xml` (common for TEI conversions of custom corpora), sentence boundaries now fall back to other segment-like TEI elements, in order of preference: `posting` (chat/CMC), `l` (verse line), `seg` (segment), `u` (utterance). This makes integrated taggers/parsers and KorAP sentence-based queries work on such corpora.
+- New `dck_sample.zip` test resource: two-text excerpt from the CC BY licensed Dortmunder Chat-Korpus with custom `cmc` tokenization/annotations (in one text the `s` spans are removed to exercise the sentence fallback)
+
+### Fixed
+
+- Corpora with custom tokenization and annotations inside the base ZIP (e.g. `cmc/morpho.xml` from TEI conversions, with no `base/tokens.xml`) are now handled correctly: the foundry is derived from the annotation folder name instead of the ZIP file name, so Krill output indexes the annotations (e.g. `cmc/p`, `cmc/l`) instead of silently dropping them, the token stream is no longer empty (`tokenSource` is set to e.g. `cmc#morpho`), and CoNLL-U output reports `# foundry = cmc` instead of `# foundry = base`
+
### Changed
- Krill metadata field names corrected by default: the misleadingly named `textClass` is now emitted as `dmozDomain` (DMOZ-based topic-domain classification) and `textDomain` as `idsColumn` (normalised newspaper column / Ressort). Pass `--legacy-field-names` to keep the historical names. Note: querying the corrected indices by the old names will require a need an accordingly configured Koral Mapper plugin to be active.
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index 6529c9b..87c11ab 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -2364,6 +2364,22 @@
return zipFileName.replace(Regex(".*\\.([^/.]+)\\.zip$"), "$1")
}
+ // In KorAP-XML, annotation files live in a folder named after their foundry
+ // (e.g. tree_tagger/morpho.xml). Corpora with custom annotations (converted
+ // from TEI with their own tokenization/POS/lemma layers) ship them inside
+ // the base ZIP in such a folder (e.g. cmc/morpho.xml), so for base ZIPs the
+ // folder name takes precedence over the foundry derived from the ZIP name.
+ private fun annotationFoundryFor(entryName: String, fileName: String, zipFoundry: String): String {
+ if (zipFoundry != "base") return zipFoundry
+ when (fileName) {
+ "morpho.xml", "dependency.xml", "sentences.xml", "constituency.xml" -> {}
+ else -> return zipFoundry
+ }
+ val parts = entryName.split('/')
+ val dir = parts.getOrNull(parts.size - 2)
+ return if (dir.isNullOrEmpty() || dir == "base") zipFoundry else dir
+ }
+
private fun getFoundryFromZipFileNames(zipFileNames: Array<String>): String {
for (zipFileName in zipFileNames) {
val foundry = getFoundryFromZipFileName(zipFileName)
@@ -3105,6 +3121,7 @@
}
// LOGGER.info("Processing file: " + zipEntry.getName())
val fileName = zipEntry.name.replace(Regex(".*?/([^/]+\\.xml)$"), "$1")
+ val annotationFoundry = annotationFoundryFor(zipEntry.name, fileName, foundry)
when (fileName) {
"data.xml" -> {
if (!lemmaOnly) {
@@ -3119,7 +3136,7 @@
val spans: NodeList = doc.getElementsByTagName("span")
if (extractAttributesRegex.isNotEmpty())
extraFeatures[docId] = extractMiscSpans(spans)
- sentences[docId] = extractSentenceSpans(spans)
+ sentences[docId] = extractSentenceSpans(spans, docId)
// For krill format, collect structural spans and base data (only from base foundry to avoid duplicates)
if (outputFormat == OutputFormat.KRILL && foundry == "base") {
@@ -3156,15 +3173,17 @@
"morpho.xml" -> {
waitForMorpho = true
fnames[docId] = zipEntry.name
- LOGGER.info("Processing morpho.xml for $docId with foundry=$foundry from ${zipEntry.name}")
+ LOGGER.info("Processing morpho.xml for $docId with foundry=$annotationFoundry from ${zipEntry.name}")
val fsSpans: NodeList = doc.getElementsByTagName("span")
val morphoSpans = extractMorphoSpans(fsSpans)
// For krill format, collect morpho data directly without using shared morpho map
if (outputFormat == OutputFormat.KRILL) {
- val morphoFoundry = getFoundryForLayer(foundry, "morpho")
+ val morphoFoundry = getFoundryForLayer(annotationFoundry, "morpho")
collectKrillMorphoDataDirect(docId, morphoFoundry, morphoSpans, "morpho")
- tokens[docId] = extractSpans(fsSpans, docId)
+ val morphoTokens = extractSpans(fsSpans, docId)
+ tokens[docId] = morphoTokens
+ collectKrillTokensFromMorpho(docId, morphoFoundry, morphoTokens)
} else {
// For other formats, use the shared morpho map
// Merge with existing morpho data (e.g., from dependency.xml)
@@ -3201,7 +3220,7 @@
// For krill format, collect dependency data directly without using shared morpho map
if (outputFormat == OutputFormat.KRILL) {
- val depFoundry = getFoundryForLayer(foundry, "dependency")
+ val depFoundry = getFoundryForLayer(annotationFoundry, "dependency")
collectKrillMorphoDataDirect(docId, depFoundry, depMap, "dependency")
} else {
// For other formats, merge dependency info into existing morpho data
@@ -3237,17 +3256,17 @@
}
"sentences.xml" -> {
- LOGGER.fine("Sentences entry foundry=$foundry for $docId from ${zipEntry.name}")
+ LOGGER.fine("Sentences entry foundry=$annotationFoundry for $docId from ${zipEntry.name}")
if (outputFormat == OutputFormat.KRILL) {
val sentenceSpans: NodeList = doc.getElementsByTagName("span")
- collectSentences(docId, foundry, sentenceSpans)
+ collectSentences(docId, annotationFoundry, sentenceSpans)
}
}
"constituency.xml" -> {
if (outputFormat == OutputFormat.KRILL || outputFormat == OutputFormat.CONLLU) {
val constituencySpans: NodeList = doc.getElementsByTagName("span")
- collectConstituency(docId, foundry, constituencySpans)
+ collectConstituency(docId, annotationFoundry, constituencySpans)
}
}
}
@@ -3287,7 +3306,7 @@
&& (extractMetadataRegex.isEmpty() || metadata[docId] != null)
) {
LOGGER.fine("All data ready for $docId, calling processText")
- tryProcessReadyText(docId, foundry)
+ tryProcessReadyText(docId, annotationFoundry)
} else {
LOGGER.fine("NOT ready to process $docId yet: textOK=${texts[docId] != null || !textRequired}, sentencesOK=${sentences[docId] != null}, tokensOK=${tokens[docId] != null}, morphoOK=${!morphoRequired || morpho[docId] != null}")
}
@@ -3401,7 +3420,8 @@
if (siglePattern != null && !Regex(siglePattern!!).containsMatchIn(docId)) return
val fileName = entryFileName
-
+ val annotationFoundry = annotationFoundryFor(zipEntry.name, fileName, foundry)
+
when (fileName) {
"data.xml" -> {
if (!lemmaOnly) {
@@ -3428,11 +3448,12 @@
"morpho.xml" -> {
fnames[docId] = zipEntry.name
val (morphoSpans, allSpans) = extractMorphoSpansStax(reader)
-
+
if (outputFormat == OutputFormat.KRILL) {
- val morphoFoundry = getFoundryForLayer(foundry, "morpho")
+ val morphoFoundry = getFoundryForLayer(annotationFoundry, "morpho")
collectKrillMorphoDataDirect(docId, morphoFoundry, morphoSpans, "morpho")
tokens[docId] = allSpans
+ collectKrillTokensFromMorpho(docId, morphoFoundry, allSpans)
} else {
val morphoMap = synchronized(morpho) {
morpho.getOrPut(docId) { morphoSpans }
@@ -3455,7 +3476,7 @@
"dependency.xml" -> {
val depMap = extractDependencySpansStax(reader)
if (outputFormat == OutputFormat.KRILL) {
- val depFoundry = getFoundryForLayer(foundry, "dependency")
+ val depFoundry = getFoundryForLayer(annotationFoundry, "dependency")
collectKrillMorphoDataDirect(docId, depFoundry, depMap, "dependency")
} else {
val morphoMap = synchronized(morpho) {
@@ -3477,11 +3498,11 @@
"sentences.xml" -> {
if (outputFormat == OutputFormat.KRILL) {
val spans = extractSpansStax(reader, docId)
- collectSentencesFromSpans(docId, foundry, spans)
+ collectSentencesFromSpans(docId, annotationFoundry, spans)
}
}
"structure.xml" -> {
- sentences[docId] = extractSentenceSpansStax(reader)
+ sentences[docId] = extractSentenceSpansStax(reader, docId)
}
}
@@ -3510,7 +3531,7 @@
&& (!finalMorphoRequired || morpho[docId] != null)
&& (extractMetadataRegex.isEmpty() || metadata[docId] != null)
) {
- tryProcessReadyText(docId, foundry)
+ tryProcessReadyText(docId, annotationFoundry)
}
} catch (e: Exception) {
@@ -4524,23 +4545,44 @@
return res
}
- private fun extractSentenceSpans(spans: NodeList): Array<Span> {
- return IntStream.range(0, spans.length).mapToObj(spans::item)
- .filter { node -> node is Element && node.getElementsByTagName("f").item(0).textContent.equals("s") }
- .map { node ->
- Span(
- Integer.parseInt((node as Element).getAttribute("from")), Integer.parseInt(node.getAttribute("to"))
- )
- }.toArray { size -> arrayOfNulls(size) }
+ // TEI elements that can stand in for sentences when a corpus has no s
+ // segmentation: chat postings, verse lines, segments, utterances (in
+ // order of preference). Without sentence spans, integrated parsers and
+ // KorAP sentence queries cannot work on such corpora.
+ private val sentenceElementFallbacks = listOf("posting", "l", "seg", "u")
+
+ private fun pickSentenceSpans(spansByName: Map<String, List<Span>>, docId: String): Array<Span> {
+ spansByName["s"]?.let { if (it.isNotEmpty()) return it.toTypedArray() }
+ for (name in sentenceElementFallbacks) {
+ val spans = spansByName[name]
+ if (!spans.isNullOrEmpty()) {
+ LOGGER.info("No sentence (s) spans in structure.xml for $docId: falling back to ${spans.size} <$name> elements")
+ return spans.toTypedArray()
+ }
+ }
+ return emptyArray()
}
- private fun extractSentenceSpansStax(reader: XMLStreamReader): Array<Span> {
- val list = ArrayList<Span>()
+ private fun extractSentenceSpans(spans: NodeList, docId: String): Array<Span> {
+ val spansByName = HashMap<String, MutableList<Span>>()
+ IntStream.range(0, spans.length).mapToObj(spans::item).forEach { node ->
+ if (node !is Element) return@forEach
+ val name = node.getElementsByTagName("f").item(0)?.textContent ?: return@forEach
+ if (name == "s" || name in sentenceElementFallbacks) {
+ spansByName.getOrPut(name) { mutableListOf() }
+ .add(Span(Integer.parseInt(node.getAttribute("from")), Integer.parseInt(node.getAttribute("to"))))
+ }
+ }
+ return pickSentenceSpans(spansByName, docId)
+ }
+
+ private fun extractSentenceSpansStax(reader: XMLStreamReader, docId: String): Array<Span> {
+ val spansByName = HashMap<String, MutableList<Span>>()
var currentFrom: Int? = null
var currentTo: Int? = null
- var isSentence = false
- var inF = false
-
+ var currentName: String? = null
+ var inNameF = false
+
while (reader.hasNext()) {
val event = reader.next()
when (event) {
@@ -4548,31 +4590,34 @@
if (reader.localName == "span") {
currentFrom = reader.getAttributeValue(null, "from")?.toIntOrNull()
currentTo = reader.getAttributeValue(null, "to")?.toIntOrNull()
- isSentence = false
- } else if (reader.localName == "f") {
- inF = true
+ currentName = null
+ } else if (reader.localName == "f" && reader.getAttributeValue(null, "name") == "name") {
+ inNameF = true
}
}
XMLStreamConstants.CHARACTERS -> {
- if (inF && reader.text.trim() == "s") {
- isSentence = true
+ if (inNameF && currentName == null) {
+ reader.text.trim().takeIf { it.isNotEmpty() }?.let { currentName = it }
}
}
XMLStreamConstants.END_ELEMENT -> {
if (reader.localName == "span") {
- if (isSentence && currentFrom != null && currentTo != null) {
- list.add(Span(currentFrom, currentTo))
+ val name = currentName
+ if (name != null && currentFrom != null && currentTo != null
+ && (name == "s" || name in sentenceElementFallbacks)
+ ) {
+ spansByName.getOrPut(name) { mutableListOf() }.add(Span(currentFrom, currentTo))
}
currentFrom = null
currentTo = null
- isSentence = false
+ currentName = null
} else if (reader.localName == "f") {
- inF = false
+ inNameF = false
}
}
}
}
- return list.toTypedArray()
+ return pickSentenceSpans(spansByName, docId)
}
private fun extractMiscSpans(spans: NodeList): MutableMap<String, String> {
@@ -5732,6 +5777,24 @@
extraFeatures.remove(docId)
}
+ // Texts with custom tokenization carry their token spans in a foundry's
+ // morpho.xml instead of base/tokens.xml (e.g. cmc/morpho.xml in CMC
+ // corpora). collectKrillBaseData only picks tokens up from tokens.xml, so
+ // register the morpho-derived spans here; real base tokens, if they arrive
+ // later, overwrite both fields via collectKrillBaseData.
+ private fun collectKrillTokensFromMorpho(docId: String, foundry: String, morphoTokens: Array<Span>) {
+ if (outputTexts.contains(docId)) return
+ val textData = krillData.getOrPut(docId) {
+ KrillJsonGenerator.KrillTextData(textId = docId)
+ }
+ synchronized(textData) {
+ if (textData.tokens == null) {
+ textData.tokens = morphoTokens
+ textData.headerMetadata.putIfAbsent("tokenSource", "$foundry#morpho")
+ }
+ }
+ }
+
// Extract the appropriate foundry name for a given annotation layer
// For combined foundries like "marmot-malt", split into morpho (marmot) and dependency (malt) parts
private fun getFoundryForLayer(foundry: String, layer: String): String {
diff --git a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/CustomTokenizationTest.kt b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/CustomTokenizationTest.kt
new file mode 100644
index 0000000..e7a62be
--- /dev/null
+++ b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/CustomTokenizationTest.kt
@@ -0,0 +1,140 @@
+package de.ids_mannheim.korapxmltools
+
+import org.junit.After
+import org.junit.Before
+import java.io.ByteArrayOutputStream
+import java.io.File
+import java.io.PrintStream
+import java.net.URL
+import java.util.zip.GZIPInputStream
+import kotlin.test.Test
+import kotlin.test.assertContains
+import kotlin.test.assertEquals
+import kotlin.test.assertFalse
+import kotlin.test.assertTrue
+
+/**
+ * Tests for corpora with custom tokenization and annotations shipped inside
+ * the base ZIP (e.g. TEI conversions with their own <w>-level POS/lemma
+ * annotations, stored in a custom foundry folder like cmc/morpho.xml instead
+ * of base/tokens.xml).
+ *
+ * The dck_sample.zip resource is a two-text excerpt from the CC BY licensed
+ * Dortmunder Chat-Korpus (DCK); in text DCK/CPR/00004 the sentence (s) spans
+ * were removed from structure.xml to exercise the sentence-segmentation
+ * fallback to <posting> elements.
+ */
+class CustomTokenizationTest {
+ private val outContent = ByteArrayOutputStream(10000000)
+ private val errContent = ByteArrayOutputStream()
+ private val originalOut: PrintStream = System.out
+ private val originalErr: PrintStream = System.err
+
+ @Before
+ fun setUpStreams() {
+ System.setOut(PrintStream(outContent))
+ System.setErr(PrintStream(errContent))
+ }
+
+ @After
+ fun restoreStreams() {
+ System.setOut(originalOut)
+ System.setErr(originalErr)
+ }
+
+ private fun loadResource(path: String): URL {
+ val resource = Thread.currentThread().contextClassLoader.getResource(path)
+ requireNotNull(resource) { "Resource $path not found" }
+ return resource
+ }
+
+ @Test
+ fun conlluUsesFoundryFromAnnotationFolderName() {
+ val args = arrayOf(loadResource("dck_sample.zip").path)
+ debug(args)
+ val output = outContent.toString()
+ assertContains(output, "# foundry = cmc")
+ assertFalse(output.contains("# foundry = base"), "Custom annotations must not be reported as base foundry")
+ assertContains(output, "cmc/morpho.xml")
+ // POS and lemma from the custom morpho.xml must survive
+ assertContains(output, "begrüssen\tbegrüßen\t_\tVVFIN")
+ }
+
+ @Test
+ fun conlluFallsBackToPostingsWithoutSentenceSpans() {
+ val args = arrayOf("-l", "info", loadResource("dck_sample.zip").path)
+ debug(args)
+ assertContains(errContent.toString(), "falling back to 145 <posting> elements")
+
+ // Text 00004 has no s spans: its tokens must still be split into
+ // sentences (one per posting) instead of one giant sentence
+ val text4 = outContent.toString().substringAfter("# text_id = DCK_CPR.00004")
+ val sentenceBreaks = text4.split("\n\n").size - 1
+ assertTrue(sentenceBreaks > 100, "Expected ~145 posting-based sentences, got $sentenceBreaks breaks")
+ }
+
+ @Test
+ fun krillCollectsCustomFoundryAnnotationsAndTokens() {
+ val outputDir = File.createTempFile("dck_krill", "").apply {
+ delete()
+ mkdirs()
+ }
+ try {
+ val args = arrayOf("-t", "krill", "-q", "-D", outputDir.path, loadResource("dck_sample.zip").path)
+ assertEquals(0, debug(args))
+
+ val tar = File(outputDir, "dck_sample.krill.tar")
+ assertTrue(tar.exists(), "Expected dck_sample.krill.tar")
+
+ val jsons = readKrillJsons(tar)
+ val json1 = jsons.getValue("DCK-CPR-00001.json")
+ val json4 = jsons.getValue("DCK-CPR-00004.json")
+
+ jsons.values.forEach { json ->
+ // Annotations must be filed under the folder-derived cmc foundry
+ assertContains(json, "cmc cmc/morpho")
+ assertContains(json, "cmc/p=tokens")
+ assertContains(json, "cmc/l=tokens")
+ assertContains(json, "\"cmc/p:")
+ assertContains(json, "\"cmc/l:")
+ // Tokens come from cmc/morpho.xml (no base/tokens.xml in the corpus)
+ assertContains(json, "\"value\":\"cmc#morpho\"")
+ }
+
+ // Surface forms must be filled from data.xml
+ assertContains(json1, "\"s:begrüssen\"")
+ assertContains(json4, "\"s:ich\"")
+
+ assertContains(json1, "-:base/sentences\$<i>184")
+ // Text 00004 has no s spans: sentence count falls back to postings
+ assertContains(json4, "-:base/sentences\$<i>145")
+ assertContains(json4, "-:base/paragraphs\$<i>145")
+ } finally {
+ outputDir.deleteRecursively()
+ }
+ }
+
+ private fun readKrillJsons(tarFile: File): Map<String, String> {
+ val extractDir = File.createTempFile("krill_extract", "").let {
+ it.delete()
+ it.mkdirs()
+ it
+ }
+ return try {
+ val tarProcess = ProcessBuilder("tar", "-xf", tarFile.path, "-C", extractDir.path)
+ .redirectErrorStream(true)
+ .start()
+ assertTrue(tarProcess.waitFor() == 0, "Tar extraction should succeed for ${tarFile.path}")
+ val jsonFiles = extractDir.listFiles()?.filter { it.name.endsWith(".json.gz") }.orEmpty()
+ assertTrue(jsonFiles.isNotEmpty(), "No JSON files found in ${tarFile.path}")
+ jsonFiles.associate { jsonFile ->
+ val jsonContent = GZIPInputStream(jsonFile.inputStream())
+ .bufferedReader()
+ .use { it.readText() }
+ jsonFile.name.removeSuffix(".gz") to jsonContent
+ }
+ } finally {
+ extractDir.deleteRecursively()
+ }
+ }
+}
diff --git a/app/src/test/resources/dck_sample.zip b/app/src/test/resources/dck_sample.zip
new file mode 100644
index 0000000..afccac5
--- /dev/null
+++ b/app/src/test/resources/dck_sample.zip
Binary files differ