Rename misleading Krill metadata fields textClass/textDomain
Emit the corrected metadata field names by default: textClass becomes
dmozDomain (DMOZ-based topic-domain classification) and textDomain becomes
idsColumn (normalised newspaper column / Ressort). The rename happens at
emission only; internal extraction/inheritance keys and the field types
(dmozDomain: keywords, idsColumn: string) are unchanged.
Add --legacy-field-names to keep the historical names for indices whose
query layer does not yet alias the old names to the new ones.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Change-Id: Ie177fe2ec88412860add8ae86715707a30b28699
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 35e8e01..46e238d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,11 @@
# Changelog
+## [Unreleased]
+
+### Changed
+
+- Krill metadata field names corrected by default: the misleadingly named `textClass` is now emitted as `dmozDomain` (DMOZ-based topic-domain classification) and `textDomain` as `idsColumn` (normalised newspaper column / Ressort). Pass `--legacy-field-names` to keep the historical names. Note: querying the corrected indices by the old names will require a need an accordingly configured Koral Mapper plugin to be active.
+
## [v3.4.0] - 2026-06-08
### Added
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index f178517..6529c9b 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -209,6 +209,12 @@
var includeNonWordTokens: Boolean = false
@Option(
+ names = ["--legacy-field-names"],
+ description = ["Keep historical Krill metadata field names instead of the corrected ones (textClass instead of dmozDomain, textDomain instead of idsColumn)."]
+ )
+ var legacyFieldNames: Boolean = false
+
+ @Option(
names = ["--lz4"],
description = ["Use LZ4 compression for Krill JSON output instead of gzip (faster but larger files)."]
)
@@ -708,6 +714,7 @@
if (useStaxTextParser) sb.appendLine(" --stax-text")
if (useLz4) sb.appendLine(" --lz4")
if (includeNonWordTokens) sb.appendLine(" --non-word-tokens")
+ if (legacyFieldNames) sb.appendLine(" --legacy-field-names")
if (sequentialInZip) sb.appendLine(" --sequential")
if (COMPATIBILITY_MODE) sb.appendLine(" COMPATIBILITY_MODE=true")
@@ -6070,7 +6077,7 @@
val byteOut = ByteArrayOutputStream()
net.jpountz.lz4.LZ4FrameOutputStream(byteOut).use { lz4Out ->
OutputStreamWriter(lz4Out, StandardCharsets.UTF_8).use { writer ->
- KrillJsonGenerator.generateTo(writer, textData, corpusMetadata, docMetadata, includeNonWordTokens)
+ KrillJsonGenerator.generateTo(writer, textData, corpusMetadata, docMetadata, includeNonWordTokens, legacyFieldNames)
}
}
Pair(fileName, byteOut.toByteArray())
@@ -6084,7 +6091,7 @@
}
gzipOut.use { gzip ->
OutputStreamWriter(gzip, StandardCharsets.UTF_8).use { writer ->
- KrillJsonGenerator.generateTo(writer, textData, corpusMetadata, docMetadata, includeNonWordTokens)
+ KrillJsonGenerator.generateTo(writer, textData, corpusMetadata, docMetadata, includeNonWordTokens, legacyFieldNames)
}
}
Pair(fileName, byteOut.toByteArray())
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
index e3fafea..cf22cbe 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/formatters/KrillJsonGenerator.kt
@@ -92,19 +92,31 @@
textData: KrillTextData,
corpusMetadata: Map<String, MutableMap<String, Any>>,
docMetadata: Map<String, MutableMap<String, Any>>,
- includeNonWordTokens: Boolean
+ includeNonWordTokens: Boolean,
+ legacyFieldNames: Boolean = false
): String {
val sb = StringBuilder()
- generateTo(sb, textData, corpusMetadata, docMetadata, includeNonWordTokens)
+ generateTo(sb, textData, corpusMetadata, docMetadata, includeNonWordTokens, legacyFieldNames)
return sb.toString()
}
+ // Krill metadata field keys that were named misleadingly early on. By default we
+ // emit the corrected names; the original names map to their proper meaning:
+ // textClass -> dmozDomain (DMOZ-based topic-domain classification)
+ // textDomain -> idsColumn (normalised newspaper column / Ressort)
+ // Pass legacyFieldNames=true to keep the historical keys.
+ private val CORRECTED_FIELD_NAMES = mapOf(
+ "textClass" to "dmozDomain",
+ "textDomain" to "idsColumn"
+ )
+
fun generateTo(
out: Appendable,
textData: KrillTextData,
corpusMetadata: Map<String, MutableMap<String, Any>>,
docMetadata: Map<String, MutableMap<String, Any>>,
- includeNonWordTokens: Boolean
+ includeNonWordTokens: Boolean,
+ legacyFieldNames: Boolean = false
) {
val sb = StringBuilder()
sb.append("{")
@@ -241,8 +253,9 @@
}
}
+ val outKey = if (legacyFieldNames) key else CORRECTED_FIELD_NAMES[key] ?: key
fields.add(jsonObject(listOf(
- "key" to jsonString(key),
+ "key" to jsonString(outKey),
"@type" to jsonString("koral:field"),
"value" to fieldValue,
"type" to jsonString(fieldType)
diff --git a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
index 20f4971..1305a86 100644
--- a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
+++ b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
@@ -1253,4 +1253,30 @@
"All categories present in the file should be indexed, in rank order"
)
}
+
+ @Test
+ fun correctedMetadataFieldNamesByDefault() {
+ val baseZip = loadResource("rei_sample.zip").path
+ val tar = ensureKrillTar("rei_corrected_names", "rei_sample.krill.tar") { outputDir ->
+ arrayOf("-t", "krill", "-q", "-D", outputDir.path, baseZip)
+ }
+ val json = readKrillJson(tar).entries.first { it.key.startsWith("REI-RBR-00473") }.value
+
+ // The misleadingly named textClass is emitted under its corrected name dmozDomain.
+ assertTrue(json.contains("\"key\":\"dmozDomain\""), "textClass should be emitted as dmozDomain by default")
+ assertFalse(json.contains("\"key\":\"textClass\""), "legacy textClass key must not appear by default")
+ }
+
+ @Test
+ fun legacyMetadataFieldNamesWithFlag() {
+ val baseZip = loadResource("rei_sample.zip").path
+ val tar = ensureKrillTar("rei_legacy_names", "rei_sample.krill.tar") { outputDir ->
+ arrayOf("-t", "krill", "-q", "--legacy-field-names", "-D", outputDir.path, baseZip)
+ }
+ val json = readKrillJson(tar).entries.first { it.key.startsWith("REI-RBR-00473") }.value
+
+ // With --legacy-field-names the historical key is kept and the corrected one is absent.
+ assertTrue(json.contains("\"key\":\"textClass\""), "--legacy-field-names should keep textClass")
+ assertFalse(json.contains("\"key\":\"dmozDomain\""), "corrected name must not appear in legacy mode")
+ }
}