Add plain TEI P5 support
Change-Id: Ia376dca8d9d8dcac99dc78ccc7dc20a053474f4a
Signed-off-by: Marc Kupietz <kupietz@ids-mannheim.de>
diff --git a/.gitignore b/.gitignore
index bd41ace..a2efa56 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@
# Source data directories
I5/
+TEI/
# Compressed archives
*.tar
diff --git a/Makefile b/Makefile
index 7f8c07b..c143581 100644
--- a/Makefile
+++ b/Makefile
@@ -5,6 +5,17 @@
I5_FILES := $(wildcard $(SRC_DIR)/*.i5.xml)
BASENAMES := $(patsubst %.i5.xml,%,$(notdir $(I5_FILES)))
+# Standard TEI P5 support
+TEI_DIR ?= TEI
+TEI_FILES := $(wildcard $(TEI_DIR)/*.xml)
+TEI_ZIP_NAME ?= tei
+TEI_FLAGS ?= --auto-textsigle 'TEI/XYZ.00001' # --xmlid-to-textsigle '([A-Z]+)\.(.*)\.([0-9]+)\.*([0-9])@$$1/$$2/$$3$$4'
+export TEI_FLAGS
+
+ifneq ($(TEI_FILES),)
+BASENAMES += $(TEI_ZIP_NAME)
+endif
+
BUILD_DIR = build
TARGET_DIR ?= ./target
MAX_THREADS ?= 8 # $(shell nproc)
@@ -25,16 +36,14 @@
index: check-src $(TARGET_DIR)/index
check-src:
- @if [ ! -d "$(SRC_DIR)" ]; then \
- echo "Error: SRC_DIR '$(SRC_DIR)' does not exist."; \
- echo "Please create it and place your .i5.xml files there,"; \
- echo "or specify a different directory using SRC_DIR variable."; \
- echo "Example: make SRC_DIR=/path/to/files"; \
+ @if [ ! -d "$(SRC_DIR)" ] && [ ! -d "$(TEI_DIR)" ]; then \
+ echo "Error: Neither SRC_DIR '$(SRC_DIR)' nor TEI_DIR '$(TEI_DIR)' exists."; \
+ echo "Please place your .i5.xml files in '$(SRC_DIR)' or your TEI .xml files in '$(TEI_DIR)'."; \
exit 1; \
fi
- @if [ -z "$$(find "$(SRC_DIR)" -maxdepth 1 -name '*.i5.xml' -print -quit)" ]; then \
- echo "Error: No .i5.xml files found in '$(SRC_DIR)'."; \
- echo "Please populate it or set SRC_DIR to a different location."; \
+ @if [ -z "$$(find "$(SRC_DIR)" -maxdepth 1 -name '*.i5.xml' -print -quit 2>/dev/null)" ] && \
+ [ -z "$$(find "$(TEI_DIR)" -maxdepth 1 -name '*.xml' -print -quit 2>/dev/null)" ]; then \
+ echo "Error: No .i5.xml files found in '$(SRC_DIR)' and no .xml files found in '$(TEI_DIR)'."; \
exit 1; \
fi
@@ -44,6 +53,11 @@
# docker run --rm $(if $(DOCKER_CPU_SHARES),--cpu-shares $(DOCKER_CPU_SHARES)) -v $(abspath $<):/input.i5.xml:ro korap/tei2korapxml:latest --progress -l warn -s -tk /input.i5.xml > $@ 2> >(tee $(@:.zip=.log) >&2)
printf "%s\t%s\n" "$$(grep -c '<idsText ' $<)" "$$(unzip -l $@ | grep data.xml | wc -l)"
+$(BUILD_DIR)/$(TEI_ZIP_NAME).zip: $(TEI_FILES)
+ mkdir -p $(BUILD_DIR)
+ docker run --rm --entrypoint /bin/sh $(if $(DOCKER_CPU_SHARES),--cpu-shares $(DOCKER_CPU_SHARES)) -v "$(abspath $(TEI_DIR)):/input:ro" korap/tei2korapxml:latest -c 'tei2korapxml -l warn -s -tk '"$$TEI_FLAGS"' /input/*.xml' > $@ 2> >(tee $(@:.zip=.log) >&2)
+ printf "%s\t%s\n" "$$(cat $^ | grep -c '<teiHeader')" "$$(unzip -l $@ | grep data.xml | wc -l)"
+
$(BUILD_DIR)/%.tree_tagger.zip: $(BUILD_DIR)/%.zip bin/korapxmltool
$(KORAPXMLTOOL) -j 1 -T treetagger -t zip --force -D $(BUILD_DIR) $<
diff --git a/Readme.md b/Readme.md
index e08d652..e31cb66 100644
--- a/Readme.md
+++ b/Readme.md
@@ -1,8 +1,8 @@
## KorAP Ingestion
-Converts I5 XML files to KorAP XML, annotates them with TreeTagger, Marmot, Malt, Spacy, CoreNLP, OpenNLP, indexes them and starts a KorAP instance.
+Converts TEI XML files to KorAP XML, annotates them with TreeTagger, Marmot, Malt, Spacy, CoreNLP, OpenNLP, indexes them and starts a KorAP instance.
-## Usage
+## Usage (TEI I5)
By default, place your `.i5.xml` files in the `I5` directory in the root of the project:
@@ -18,6 +18,40 @@
Then open http://localhost:64543 in your browser.
+### Standard TEI P5 Support
+
+Standard TEI P5 files (which typically contain one text per file) can be batch-converted together. By default, place your `.xml` files in the `TEI` directory in the root of the project:
+
+```bash
+make
+```
+
+All files in the `TEI` directory will be packaged into a single KorAP-XML zip archive.
+
+You can specify a different source directory using the `TEI_DIR` variable:
+
+```bash
+make TEI_DIR=/path/to/your/tei/files
+```
+
+By default, texts are assigned automatic three-part KorAP/DeReKo sigles starting from:
+```
+--auto-textsigle 'TEI/XYZ.00001'
+```
+You can customize this or define a mapping from the `xml:id` of each text to a sigle. For example, if your XML files have IDs like this:
+
+```xml
+<?xml version="1.0" encoding="UTF-8"?>
+<TEI xmlns="http://www.tei-c.org/ns/1.0" xml:id="SK.UL.1981.1" version="3.4.0">
+ <teiHeader>
+```
+
+You can extract the three-part sigle (e.g., `SK/UL/19811`) by overriding the `TEI_FLAGS` variable with a matching regular expression:
+
+```bash
+make TEI_FLAGS='--xmlid-to-textsigle '\''([A-Z]+)\.(.*)\.([0-9]+)\.*([0-9])@$$1/$$2/$$3$$4'\'''
+```
+
## Prerequisites
* Java JRE 21 and