Handle ZIP64 annotation archives in text exports
Retry streaming word2vec and NOW inputs with Commons Compress when the JDK ZIP reader rejects their ZIP64 metadata. Propagate parallel ZIP worker failures instead of returning success with empty output.
Change-Id: I20e7a808c746e9f04533ec1c3c6dd23f9cb8c1f8
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f4bf0cd..7aee391 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -11,6 +11,7 @@
### Fixed
- Stabilized large `word2vec`/`NOW` exports: ZIP readers now share one bounded XML-entry backlog instead of multiplying the backlog by every open ZIP, plain-output document claims are released when each ZIP closes instead of growing for the whole corpus, annotation ZIPs are not re-reading the same base ZIP for surface output, and unused late annotation layers no longer repopulate cleaned document state.
+- `word2vec`/`NOW` lemma exports now fall back to Apache Commons Compress when Java's ZIP reader rejects ZIP64 metadata in annotation archives such as `.tree_tagger.zip`. Parallel ZIP-reader failures are also propagated to the main command instead of being silently discarded and yielding an empty output file with exit status 0.
## [v4.1.0] - 2026-06-19
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index 4002198..fcf32f0 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -37,6 +37,7 @@
import java.util.regex.Pattern
import java.util.stream.IntStream
import java.util.zip.GZIPOutputStream
+import java.util.zip.ZipException
import java.util.zip.ZipFile
import kotlin.text.Charsets
import me.tongfei.progressbar.ProgressBar
@@ -2125,9 +2126,10 @@
val queue: java.util.concurrent.BlockingQueue<String> = java.util.concurrent.LinkedBlockingQueue()
zips.forEach { queue.put(it) }
val executor = Executors.newFixedThreadPool(parallelism)
+ val completions = java.util.concurrent.ExecutorCompletionService<Unit>(executor)
val active = java.util.concurrent.atomic.AtomicInteger(0)
repeat(parallelism) {
- executor.submit {
+ completions.submit(Callable {
active.incrementAndGet()
val activeNow = activeZipWorkers.incrementAndGet()
peakActiveZipWorkers.accumulateAndGet(activeNow, ::maxOf)
@@ -2154,13 +2156,23 @@
active.decrementAndGet()
activeZipWorkers.decrementAndGet()
}
- }
+ Unit
+ })
}
executor.shutdown()
try {
- executor.awaitTermination(7, java.util.concurrent.TimeUnit.DAYS)
+ repeat(parallelism) {
+ try {
+ completions.take().get()
+ } catch (e: java.util.concurrent.ExecutionException) {
+ executor.shutdownNow()
+ throw (e.cause ?: e)
+ }
+ }
} catch (ie: InterruptedException) {
+ executor.shutdownNow()
Thread.currentThread().interrupt()
+ throw ie
}
}
@@ -2798,6 +2810,35 @@
return zipFile
}
+ private fun processTextStreamingZip(
+ path: String,
+ foundry: String,
+ waitForMorpho: Boolean
+ ) {
+ val javaZip = try {
+ openJavaZipFile(path)
+ } catch (e: ZipException) {
+ LOGGER.warning(
+ "Java ZIP reader rejected $path (${e.message}); " +
+ "retrying with Apache Commons Compress"
+ )
+ null
+ }
+
+ if (javaZip != null) {
+ javaZip.use { zipFile ->
+ LOGGER.info("Using ${textStreamingModeLabel()} streaming mode for $path: archive-order entries, no text-ID sorting")
+ processZipEntriesStreaming(zipFile, path, foundry, waitForMorpho)
+ }
+ return
+ }
+
+ openZipFile(path).use { zipFile ->
+ LOGGER.info("Using ${textStreamingModeLabel()} streaming mode for $path with Apache ZIP reader: archive-order entries, no text-ID sorting")
+ processZipEntriesStreaming(zipFile, path, foundry, waitForMorpho)
+ }
+ }
+
private fun getFoundryFromZipFileName(zipFileName: String): String {
if (!zipFileName.matches(Regex(".*\\.([^/.]+)\\.zip$"))) {
return "base"
@@ -2971,10 +3012,7 @@
val requireRelatedMorpho =
if (outputFormat == OutputFormat.WORD2VEC || outputFormat == OutputFormat.NOW) useLemma else true
if (useJavaZipForTextStreaming()) {
- openJavaZipFile(zip).use { zipFile ->
- LOGGER.info("Using ${textStreamingModeLabel()} streaming mode for $zip: archive-order entries, no text-ID sorting")
- processZipEntriesStreaming(zipFile, zip, zipFoundry, requireRelatedMorpho)
- }
+ processTextStreamingZip(zip, zipFoundry, requireRelatedMorpho)
} else {
openZipFile(zip).use { zipFile ->
processZipEntriesWithPool(zipFile, zip, zipFoundry, requireRelatedMorpho)
@@ -2986,10 +3024,7 @@
try {
// If no corresponding base ZIP exists, this IS the base ZIP
if (useJavaZipForTextStreaming()) {
- openJavaZipFile(zipFilePath).use { zipFile ->
- LOGGER.info("Using ${textStreamingModeLabel()} streaming mode for $zipFilePath: archive-order entries, no text-ID sorting")
- processZipEntriesStreaming(zipFile, zipFilePath, foundry, false)
- }
+ processTextStreamingZip(zipFilePath, foundry, false)
} else {
openZipFile(zipFilePath).use { zipFile ->
LOGGER.fine("Calling processZipEntriesWithPool, foundry=$foundry")
@@ -3000,6 +3035,7 @@
} catch (e: Exception) {
LOGGER.severe("Error processing ZIP: ${e.message}")
e.printStackTrace()
+ throw e
}
}
// Don't close the ZIP here if using external annotation - it will be closed after worker pool finishes
@@ -3039,10 +3075,7 @@
val requireRelatedMorpho =
if (outputFormat == OutputFormat.WORD2VEC || outputFormat == OutputFormat.NOW) useLemma else true
if (useJavaZipForTextStreaming()) {
- openJavaZipFile(zip).use { zipFile ->
- LOGGER.info("Using ${textStreamingModeLabel()} streaming mode for $zip: archive-order entries, no text-ID sorting")
- processZipEntriesStreaming(zipFile, zip, zipFoundry, requireRelatedMorpho)
- }
+ processTextStreamingZip(zip, zipFoundry, requireRelatedMorpho)
} else {
openZipFile(zip).use { zipFile ->
// Iterate entries sorted by text ID to ensure consistent processing order
@@ -3057,10 +3090,7 @@
}
} else {
if (useJavaZipForTextStreaming()) {
- openJavaZipFile(zipFilePath).use { zipFile ->
- LOGGER.info("Using ${textStreamingModeLabel()} streaming mode for $zipFilePath: archive-order entries, no text-ID sorting")
- processZipEntriesStreaming(zipFile, zipFilePath, foundry, false)
- }
+ processTextStreamingZip(zipFilePath, foundry, false)
} else {
openZipFile(zipFilePath).use { zipFile ->
zipFile.entries.toList()