)]}'
{
  "commit": "2d421910c2bebe7239c0bd011631ba3a55d62008",
  "tree": "b8cced9b969285d1db4dc91cbdd6e7ce594132fd",
  "parents": [
    "800d9b7b9cc945dd7d4083bf775be1bbf0cbe49c"
  ],
  "author": {
    "name": "Marc Kupietz",
    "email": "kupietz@ids-mannheim.de",
    "time": "Wed Jun 10 07:25:33 2026 +0200"
  },
  "committer": {
    "name": "Marc Kupietz",
    "email": "kupietz@ids-mannheim.de",
    "time": "Wed Jun 10 07:25:33 2026 +0200"
  },
  "message": "Fix worker-pool streaming deadlock on large corpora\n\nAnnotate and flush per document instead of buffering 20000 sentences with\nno flush. parse_spacy_pipe.py now treats korapxmltool\u0027s \u0027# eot\u0027/\u0027# eof\u0027\nlines as document delimiters (rather than swallowing them as CoNLL-U\ncomments), emits the matching marker after each document, and flushes\nstdout. This lets korapxmltool\u0027s AnnotationWorkerPool deliver each result\nand release its bounded in-flight buffer slot.\n\nPreviously nothing was emitted until the process exited at stdin EOF, so\nthe in-flight buffer filled and the producer blocked in\nbufferedTaskPermits.acquire while the parser blocked reading stdin for a\nfull chunk - a deadlock with idle CPU and zero progress that only appeared\non corpora with many small documents (small ones fit inside the buffer).\n\nInputs without protocol markers (a CoNLL-U file piped in directly) are\nstill read in SPACY_CHUNK_SIZE-bounded blocks and now stream their output.\n\nVerified end-to-end with \u0027korapxmltool -P spacy\u0027 on zca24.zip: progress\nstreams 0-\u003e95 to completion, morpho/dependency output is correct, and no\nmarkers leak into the annotations.\n\nCo-Authored-By: Claude Opus 4.8 \u003cnoreply@anthropic.com\u003e\nChange-Id: I07d18c54e6e3238758b2c4a885ceabd428ec3f82\n",
  "tree_diff": [
    {
      "type": "modify",
      "old_id": "16fd4e1b189dbe9317ba50146a4317e60487add3",
      "old_mode": 33188,
      "old_path": "CHANGELOG.md",
      "new_id": "3a39137e9b858a438f2f464538d1ead581dfcd10",
      "new_mode": 33188,
      "new_path": "CHANGELOG.md"
    },
    {
      "type": "modify",
      "old_id": "87366698e0821b696554b36d0cadd79ba6843769",
      "old_mode": 33188,
      "old_path": "systems/parse_spacy_pipe.py",
      "new_id": "043270991a9eafea730b5495442bc508b1493154",
      "new_mode": 33188,
      "new_path": "systems/parse_spacy_pipe.py"
    }
  ]
}