When zip output log to .log files
Change-Id: I1378d2acc1d018ad6a2fe79247ff7b9fb41e3880
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/AnnotationWorkerPool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/AnnotationWorkerPool.kt
index 9fd108f..85e7ef3 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/AnnotationWorkerPool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/AnnotationWorkerPool.kt
@@ -17,13 +17,22 @@
private val command: String,
private val numWorkers: Int,
private val LOGGER: Logger,
- private val outputHandler: ((String, AnnotationTask?) -> Unit)? = null
+ private val outputHandler: ((String, AnnotationTask?) -> Unit)? = null,
+ private val stderrLogPath: String? = null
) {
private val queue: BlockingQueue<AnnotationTask> = LinkedBlockingQueue()
private val threads = mutableListOf<Thread>()
private val threadCount = AtomicInteger(0)
private val threadsLock = Any()
private val pendingOutputHandlers = AtomicInteger(0) // Track pending outputHandler calls
+ private val stderrWriter: PrintWriter? = try {
+ stderrLogPath?.let { path ->
+ PrintWriter(BufferedWriter(FileWriter(path, true)), true)
+ }
+ } catch (e: IOException) {
+ LOGGER.warning("Failed to open stderr log file '$stderrLogPath': ${e.message}")
+ null
+ }
data class AnnotationTask(val text: String, val docId: String?, val entryPath: String?)
@@ -57,8 +66,9 @@
}
val process = ProcessBuilder("/bin/sh", "-c", command)
- .redirectOutput(ProcessBuilder.Redirect.PIPE).redirectInput(ProcessBuilder.Redirect.PIPE)
- .redirectError(ProcessBuilder.Redirect.INHERIT)
+ .redirectOutput(ProcessBuilder.Redirect.PIPE)
+ .redirectInput(ProcessBuilder.Redirect.PIPE)
+ .redirectError(ProcessBuilder.Redirect.PIPE)
.start()
if (process.outputStream == null) {
@@ -72,6 +82,7 @@
// Using try-with-resources for streams to ensure they are closed
process.outputStream.buffered(BUFFER_SIZE).use { procOutStream ->
process.inputStream.buffered(BUFFER_SIZE).use { procInStream ->
+ val procErrStream = process.errorStream.buffered(BUFFER_SIZE)
val coroutineScope = CoroutineScope(Dispatchers.IO + Job()) // Ensure Job can be cancelled
var inputGotEof = false // Specific to this worker's process interaction
@@ -120,7 +131,7 @@
}
}
- // Reader coroutine
+ // Reader coroutine (stdout)
coroutineScope.launch {
val output = StringBuilder()
var lastLineWasEmpty = false
@@ -246,6 +257,29 @@
}
}
+ // Stderr reader coroutine
+ coroutineScope.launch {
+ try {
+ procErrStream.bufferedReader().use { errReader ->
+ var line: String?
+ while (true) {
+ line = errReader.readLine()
+ if (line == null) break
+ stderrWriter?.let { w ->
+ synchronized(w) {
+ w.println("[ext-$workerIndex] $line")
+ w.flush()
+ }
+ } ?: run {
+ LOGGER.warning("[ext-$workerIndex][stderr] $line")
+ }
+ }
+ }
+ } catch (e: Exception) {
+ LOGGER.fine("Worker $workerIndex: stderr reader finished: ${e.message}")
+ }
+ }
+
// Wait for coroutines to complete
try {
runBlocking {
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index 4d69378..2c09c29 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -370,40 +370,64 @@
if (annotateWith.isNotEmpty()) {
// Detect external foundry label once from annotateWith command
externalFoundry = detectFoundryFromAnnotateCmd(annotateWith)
- // Initialize ZIP output stream BEFORE creating worker pool, if needed
- if (outputFormat == OutputFormat.KORAPXML) {
- // Determine output filename
- val inputZipPath = args[0] // First ZIP file
+ // Initialize ZIP output stream BEFORE creating worker pool, if needed
+ if (outputFormat == OutputFormat.KORAPXML) {
+ // Determine output filename
+ val inputZipPath = args[0] // First ZIP file
val targetFoundry = externalFoundry ?: "annotated"
- val outputMorphoZipFileName = inputZipPath.replace(Regex("\\.zip$"), ".".plus(targetFoundry).plus(".zip"))
- LOGGER.info("Initializing output ZIP: $outputMorphoZipFileName (from input: $inputZipPath, foundry: $targetFoundry)")
+ val outputMorphoZipFileName = inputZipPath.replace(Regex("\\.zip$"), ".".plus(targetFoundry).plus(".zip"))
+ LOGGER.info("Initializing output ZIP: $outputMorphoZipFileName (from input: $inputZipPath, foundry: $targetFoundry)")
- if (File(outputMorphoZipFileName).exists() && !overwrite) {
- LOGGER.severe("Output file $outputMorphoZipFileName already exists. Use --overwrite to overwrite.")
- exitProcess(1)
- }
+ // Prepare per-output log file
+ val logFilePath = outputMorphoZipFileName.replace(Regex("\\.zip$"), ".log")
+ val fileHandler = java.util.logging.FileHandler(logFilePath, true)
+ fileHandler.formatter = ColoredFormatter()
+ LOGGER.addHandler(fileHandler)
+ LOGGER.info("Logging redirected to: $logFilePath")
+ // Mirror System.err to the same log file for the duration
+ val errPs = java.io.PrintStream(java.io.BufferedOutputStream(java.io.FileOutputStream(logFilePath, true)), true)
+ val oldErr = System.err
+ System.setErr(errPs)
- // Delete old file if it exists
- if (File(outputMorphoZipFileName).exists()) {
- LOGGER.info("Deleting existing file: $outputMorphoZipFileName")
- File(outputMorphoZipFileName).delete()
- }
+ if (File(outputMorphoZipFileName).exists() && !overwrite) {
+ LOGGER.severe("Output file $outputMorphoZipFileName already exists. Use --overwrite to overwrite.")
+ exitProcess(1)
+ }
- dbFactory = DocumentBuilderFactory.newInstance()
- dBuilder = dbFactory!!.newDocumentBuilder()
- val fileOutputStream = FileOutputStream(outputMorphoZipFileName)
- morphoZipOutputStream = ZipArchiveOutputStream(fileOutputStream).apply {
- setUseZip64(Zip64Mode.Always)
- }
- LOGGER.info("Initialized morphoZipOutputStream for external annotation to: $outputMorphoZipFileName")
- }
+ // Delete old file if it exists
+ if (File(outputMorphoZipFileName).exists()) {
+ LOGGER.info("Deleting existing file: $outputMorphoZipFileName")
+ File(outputMorphoZipFileName).delete()
+ }
+
+ dbFactory = DocumentBuilderFactory.newInstance()
+ dBuilder = dbFactory!!.newDocumentBuilder()
+ val fileOutputStream = FileOutputStream(outputMorphoZipFileName)
+ morphoZipOutputStream = ZipArchiveOutputStream(fileOutputStream).apply {
+ setUseZip64(Zip64Mode.Always)
+ }
+ LOGGER.info("Initialized morphoZipOutputStream for external annotation to: $outputMorphoZipFileName")
+
+ // Ensure we restore System.err and remove file handler at the end of processing (shutdown hook)
+ Runtime.getRuntime().addShutdownHook(Thread {
+ try {
+ LOGGER.info("Shutting down; closing per-zip log handler")
+ LOGGER.removeHandler(fileHandler)
+ fileHandler.close()
+ } catch (_: Exception) {}
+ try { System.setErr(oldErr) } catch (_: Exception) {}
+ try { errPs.close() } catch (_: Exception) {}
+ })
+ }
if (outputFormat == OutputFormat.KORAPXML) {
// For ZIP output with external annotation, we need a custom handler
- annotationWorkerPool = AnnotationWorkerPool(annotateWith, maxThreads, LOGGER) { annotatedConllu, task ->
- parseAndWriteAnnotatedConllu(annotatedConllu, task)
- }
+ val currentZipPath = args[0].replace(Regex("\\.zip$"), "." + (externalFoundry ?: "annotated") + ".zip")
+ val currentLog = currentZipPath.replace(Regex("\\.zip$"), ".log")
+ annotationWorkerPool = AnnotationWorkerPool(annotateWith, maxThreads, LOGGER, { annotatedConllu, task ->
+ parseAndWriteAnnotatedConllu(annotatedConllu, task)
+ }, stderrLogPath = currentLog)
} else {
annotationWorkerPool = AnnotationWorkerPool(annotateWith, maxThreads, LOGGER, null)
}