Always emit Krill availability, inherited corpus->doc->text, default unknown
The availability (licence) field is legally required for a KorAP instance to
serve a text, so every Krill document must carry it. resolveHeaderMetadata
already inherits metadata corpus -> doc -> text (a lower level overriding a
higher one); this makes availability follow the same path and, when no value
is present at any level, defaults it to "unknown" so the field is never
missing. Each defaulted text is logged at FINE and counted, and a single
summary warning reports the affected text count at the end of Krill output.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Change-Id: Ia588c27f8cfeccb320841c8876de29c83193018c
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 86bf61d..5b4233c 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@
- Sentence segmentation fallback: when a corpus has no `s` spans in `structure.xml` (common for TEI conversions of custom corpora), sentence boundaries now fall back to other segment-like TEI elements, in order of preference: `posting` (chat/CMC), `l` (verse line), `seg` (segment), `u` (utterance). This makes integrated taggers/parsers and KorAP sentence-based queries work on such corpora.
- New `dck_sample.zip` test resource: two-text excerpt from the CC BY licensed Dortmunder Chat-Korpus with custom `cmc` tokenization/annotations (in one text the `s` spans are removed to exercise the sentence fallback)
- Krill output now indexes extra inline `<w>`-level annotations from TEI-derived corpora as their own token layers, alongside POS (`p`) and lemma (`l`): `norm`, `orig`, `phon` and `trans` (read from the correspondingly named `<f>` features in `morpho.xml`). Each becomes a `foundry/<name>=tokens` layer with `foundry/<name>:<value>` terms (original casing preserved), grouped under the `morpho` foundry. The set is intentionally restricted to these four names to keep the Krill hot path fast. These layers are Krill-only and do not affect CoNLL-U output.
+- Krill output now always emits an `availability` field, which is legally required for a KorAP instance to serve a text. It is inherited from the corpus or document header when not set on the text (a lower level overrides a higher one); if no value is found at any level it defaults to `unknown`, logged per text at `FINE` and summarised in a single closing warning with the affected text count.
### Fixed
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index 69b655b..d3e3b39 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -1319,6 +1319,9 @@
var krillOutputCount = java.util.concurrent.atomic.AtomicInteger(0)
// Texts dropped from Krill output because they contain no tokens (see enqueueKrillCompression).
val krillEmptyTextCount = java.util.concurrent.atomic.AtomicInteger(0)
+ // Texts for which no availability metadata could be resolved and that were
+ // defaulted to "unknown" (see applyInheritedKrillMetadata).
+ val krillMissingAvailabilityCount = java.util.concurrent.atomic.AtomicInteger(0)
private val krillPeakRawPending = AtomicInteger(0)
private val krillPeakCompressedPending = AtomicInteger(0)
private val krillPeakCompressionInFlight = AtomicInteger(0)
@@ -1956,6 +1959,10 @@
if (emptyDropped > 0) {
LOGGER.warning("Dropped $emptyDropped empty text(s) with no tokens from Krill output")
}
+ val missingAvailability = krillMissingAvailabilityCount.get()
+ if (missingAvailability > 0) {
+ LOGGER.warning("Set availability to \"unknown\" for $missingAvailability text(s) with no availability metadata in their text, doc or corpus header")
+ }
LOGGER.info("Closed krill TAR file: $krillOutputFileName (total texts output: $krillOutputCount)")
} catch (e: Exception) {
LOGGER.severe("ERROR generating krill output: ${e.message}")
@@ -6189,6 +6196,20 @@
corpusMetadata = corpusMetadata,
docMetadata = docMetadata
)
+
+ // availability is legally crucial for a KorAP instance to serve a text.
+ // resolveHeaderMetadata already inherits it corpus -> doc -> text (a lower
+ // level overrides a higher one); if it is still missing, default to
+ // "unknown" so the field is always present, and record it for a summary
+ // warning. The per-text detail is logged at FINE to avoid flooding the
+ // log when a whole corpus lacks the field.
+ val availability = resolvedMetadata["availability"]
+ if (availability == null || (availability is String && availability.isBlank())) {
+ resolvedMetadata["availability"] = "unknown"
+ krillMissingAvailabilityCount.incrementAndGet()
+ LOGGER.fine("No availability metadata for $textId (absent from text, doc and corpus header); set to \"unknown\"")
+ }
+
textData.headerMetadata.clear()
textData.headerMetadata.putAll(resolvedMetadata)
diff --git a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
index 9cbc88b..89f229b 100644
--- a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
+++ b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
@@ -134,6 +134,14 @@
return tool.corpusMetadata.getValue(corpusSigle)
}
+ private fun applyInheritedKrillMetadata(tool: KorapXmlTool, textId: String, textData: KrillJsonGenerator.KrillTextData) {
+ val method = KorapXmlTool::class.java.getDeclaredMethod(
+ "applyInheritedKrillMetadata", String::class.java, KrillJsonGenerator.KrillTextData::class.java
+ )
+ method.isAccessible = true
+ method.invoke(tool, textId, textData)
+ }
+
private fun krillFieldValue(json: String, fieldName: String): String? =
Regex(
""""key"\s*:\s*"$fieldName".*?"value"\s*:\s*"([^"]*)"""",
@@ -591,6 +599,29 @@
}
@Test
+ fun availabilityIsInheritedFromCorpusWhenAbsentOnText() {
+ val tool = KorapXmlTool()
+ tool.corpusMetadata["TEST"] = mutableMapOf<String, Any>("availability" to "CC-BY-SA")
+ val textData = KrillJsonGenerator.KrillTextData(textId = "TEST_DOC.1")
+
+ applyInheritedKrillMetadata(tool, "TEST_DOC.1", textData)
+
+ assertEquals("CC-BY-SA", textData.headerMetadata["availability"])
+ assertEquals(0, tool.krillMissingAvailabilityCount.get(), "Inherited availability must not count as missing")
+ }
+
+ @Test
+ fun availabilityDefaultsToUnknownAndIsCountedWhenMissingEverywhere() {
+ val tool = KorapXmlTool()
+ val textData = KrillJsonGenerator.KrillTextData(textId = "TEST_DOC.1")
+
+ applyInheritedKrillMetadata(tool, "TEST_DOC.1", textData)
+
+ assertEquals("unknown", textData.headerMetadata["availability"])
+ assertEquals(1, tool.krillMissingAvailabilityCount.get(), "Missing availability must be counted for the summary warning")
+ }
+
+ @Test
fun docAndCorpusMetadataCollectorsExposeCommonInheritedFields() {
val tool = KorapXmlTool()