Drop empty texts (0 tokens) from Krill output (#46)
Krill should not index a text without at least one token, so emitting
such a text only produces an empty, unusable document. Texts with no
tokens are now skipped with a per-text warning and a summary count at
the end of the run; texts with at least one token are kept.
The check is applied at the single chokepoint every Krill text passes
through before being written (enqueueKrillCompression), with a defensive
guard on the direct-write fallback (outputKrillText). A new
dropKrillText helper removes the text from all tracking structures so
the finalization pass skips it too.
Adds m21_empty_sample.zip regression fixture (one empty text, one
single-token text) and a test asserting the empty text is dropped while
the single-token text is kept.
Resolves รค46
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Change-Id: I2786557463249b105a4418d20dfbb736bfadc942
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8177fef..7673eb6 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -9,6 +9,7 @@
### Fixed
+- Krill output now drops texts that contain no tokens instead of emitting empty, unindexable documents ([#46](https://github.com/KorAP/korapxmltool/issues/46)). Such texts (e.g. articles with empty `data.xml`/`base/tokens.xml`) are skipped with a per-text warning and a summary count; texts with at least one token are kept. New `m21_empty_sample.zip` regression fixture (one empty text, one single-token text).
- Corpora with custom tokenization and annotations inside the base ZIP (e.g. `cmc/morpho.xml` from TEI conversions, with no `base/tokens.xml`) are now handled correctly: the foundry is derived from the annotation folder name instead of the ZIP file name, so Krill output indexes the annotations (e.g. `cmc/p`, `cmc/l`) instead of silently dropping them, the token stream is no longer empty (`tokenSource` is set to e.g. `cmc#morpho`), and CoNLL-U output reports `# foundry = cmc` instead of `# foundry = base`
### Changed
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index 87c11ab..22bd2a0 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -1294,6 +1294,8 @@
val expectedFoundries: MutableSet<String> = mutableSetOf("base")
val processedFoundries: MutableSet<String> = mutableSetOf()
var krillOutputCount = java.util.concurrent.atomic.AtomicInteger(0)
+ // Texts dropped from Krill output because they contain no tokens (see enqueueKrillCompression).
+ val krillEmptyTextCount = java.util.concurrent.atomic.AtomicInteger(0)
private val krillPeakRawPending = AtomicInteger(0)
private val krillPeakCompressedPending = AtomicInteger(0)
private val krillPeakCompressionInFlight = AtomicInteger(0)
@@ -1927,6 +1929,10 @@
// Close incremental progress bar if it was initialized
incrementalProgressBar?.close()
+ val emptyDropped = krillEmptyTextCount.get()
+ if (emptyDropped > 0) {
+ LOGGER.warning("Dropped $emptyDropped empty text(s) with no tokens from Krill output")
+ }
LOGGER.info("Closed krill TAR file: $krillOutputFileName (total texts output: $krillOutputCount)")
} catch (e: Exception) {
LOGGER.severe("ERROR generating krill output: ${e.message}")
@@ -6107,9 +6113,36 @@
}
}
+ // Remove a text from every Krill tracking structure without writing it. Used for empty
+ // texts that must not appear in the output TAR. Marking it in outputTexts ensures the
+ // incremental writer and the finalization pass both treat it as already handled.
+ private fun dropKrillText(textId: String) {
+ outputTexts.add(textId)
+ krillData.remove(textId)
+ krillCompressedData.remove(textId)
+ krillCompressionFutures.remove(textId)
+ krillCompressionStartNanos.remove(textId)
+ val relevantZips = zipInventory.filter { (_, texts) -> texts.contains(textId) }.keys
+ relevantZips.forEach { path ->
+ zipInventory[path]?.remove(textId)
+ processedTextsPerZip[path]?.remove(textId)
+ }
+ }
+
private fun enqueueKrillCompression(textId: String, textData: KrillJsonGenerator.KrillTextData) {
if (krillCompressedData.containsKey(textId)) return
+ // Drop texts that contain no tokens: Krill cannot index a text without at least
+ // one token, so emitting it would only produce an empty, unusable document. We log
+ // a warning and remove it from all tracking so the finalization pass skips it too.
+ val tokenCount = textData.tokens?.size ?: 0
+ if (tokenCount == 0) {
+ LOGGER.warning("Skipping text $textId: no tokens (empty text)")
+ krillEmptyTextCount.incrementAndGet()
+ dropKrillText(textId)
+ return
+ }
+
val executor = compressionExecutor
val future = if (executor != null && !executor.isShutdown) {
executor.submit {
@@ -6420,6 +6453,14 @@
// Output a single text to Krill TAR (thread-safe)
private fun outputKrillText(textId: String, textData: KrillJsonGenerator.KrillTextData) {
+ // Never emit a text without tokens: Krill cannot index an empty document.
+ if ((textData.tokens?.size ?: 0) == 0) {
+ LOGGER.warning("Skipping text $textId: no tokens (empty text)")
+ krillEmptyTextCount.incrementAndGet()
+ dropKrillText(textId)
+ freeTextMemory(textId)
+ return
+ }
try {
val (jsonFileName, compressedData) = synchronized(textData) {
applyInheritedKrillMetadata(textId, textData)
diff --git a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
index 1305a86..1316c14 100644
--- a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
+++ b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
@@ -1279,4 +1279,40 @@
assertTrue(json.contains("\"key\":\"textClass\""), "--legacy-field-names should keep textClass")
assertFalse(json.contains("\"key\":\"dmozDomain\""), "corrected name must not appear in legacy mode")
}
+
+ /**
+ * Regression test for https://github.com/KorAP/korapxmltool/issues/46
+ *
+ * Krill should not index a text that has no tokens, so empty texts must be dropped (with a
+ * warning) instead of producing unusable empty documents. The fixture contains two texts:
+ * - M21/FEB.04748: an empty text with 0 tokens -> must be dropped
+ * - M21/FEB.04749: a text with exactly 1 token -> must be kept
+ */
+ @Test
+ fun krillDropsEmptyTextsButKeepsSingleTokenTexts() {
+ val baseZip = loadResource("m21_empty_sample.zip").path
+ val tar = ensureKrillTar("m21_empty_sample", "m21_empty_sample.krill.tar") { outputDir ->
+ arrayOf("-t", "krill", "-q", "-l", "info", "-D", outputDir.path, baseZip)
+ }
+
+ val tarListProcess = ProcessBuilder("tar", "-tf", tar.path).redirectErrorStream(true).start()
+ val entries = tarListProcess.inputStream.bufferedReader().readLines()
+ assertTrue(tarListProcess.waitFor() == 0)
+
+ val textIds = entries.filter { it.endsWith(".json.gz") }.map { it.removeSuffix(".json.gz") }
+
+ // The single-token text is kept; the empty text is dropped.
+ assertTrue(textIds.any { it.endsWith("M21-FEB-04749") }, "1-token text must be kept, got: $textIds")
+ assertFalse(textIds.any { it.endsWith("M21-FEB-04748") }, "empty text must be dropped, got: $textIds")
+ assertEquals(1, textIds.size, "exactly one text expected in the TAR, got: $textIds")
+
+ // The drop is reported in the run log.
+ val logFile = File(tar.path.replace(Regex("\\.tar$"), ".log"))
+ assertTrue(logFile.exists(), "krill run log should exist")
+ val log = logFile.readText()
+ assertTrue(
+ log.contains("Skipping text M21_FEB.04748: no tokens"),
+ "log should warn about the dropped empty text"
+ )
+ }
}
diff --git a/app/src/test/resources/m21_empty_sample.zip b/app/src/test/resources/m21_empty_sample.zip
new file mode 100644
index 0000000..6481355
--- /dev/null
+++ b/app/src/test/resources/m21_empty_sample.zip
Binary files differ