CI: Reduce threads for maltparsing
otherwise oom error
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index f8ecbc8..042feb6 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -92,8 +92,8 @@
- start_section building_index "Building index"
- touch target/*.zip # ignore timestamps in make
- sleep 1
- - touch target/*.ud.zip
- - make -j $(nproc) target/dnb.index.tar.xz
+ - touch target/*.*.zip
+ - MAX_THREADS=2 make -j $(nproc) target/dnb.index.tar.xz
- end_section building_krill
artifacts:
paths:
diff --git a/Makefile b/Makefile
index eb0aca7..6f1c77f 100644
--- a/Makefile
+++ b/Makefile
@@ -4,6 +4,7 @@
DEPLOY_HOST ?= compute.ids-mannheim.de
DEPLOY_USER ?= korap
DEPLOY_PATH ?= /export/netapp/korap4dnb
+MAX_THREADS ?= $(shell nproc)
.PHONY: all clean test krill index deploy server-log server-status
@@ -54,7 +55,7 @@
wget -O $@ https://corpora.ids-mannheim.de/tools/$@
%.marmot-malt.zip: %.zip models/de.marmot models/german.mco
- $(KORAPXML2CONLLU) -t marmot:models/de.marmot -P malt:models/german.mco $< | tee $(TARGET_DIR)/dnb.marmot-malt.conllu | conllu2korapxml > $@
+ $(KORAPXML2CONLLU) -T $(MAX_THREADS) -t marmot:models/de.marmot -P malt:models/german.mco $< | tee $(TARGET_DIR)/dnb.marmot-malt.conllu | conllu2korapxml > $@
%.ud.zip: %.zip
$(KORAPXML2CONLLU) $< | pv | ./scripts/udpipe2 | conllu2korapxml > $@