KP-10981 don't extract zip, loop through all VRTs
diff --git a/klk2eureco.sh b/klk2eureco.sh
index 16e85ad..0a466e5 100755
--- a/klk2eureco.sh
+++ b/klk2eureco.sh
@@ -6,7 +6,12 @@
CORPUS="$EURECO/klk-fi-v2-vrt"
TEIDTD="$EURECO/tei/tei_all.dtd"
-DAT="$CORPUS/data" # input vrt files of klk-fi by year
+SCRIPTDIR="$(cd "$(dirname "$0")" && pwd)"
+ZIPFILE=`ls "$SCRIPTDIR"/workdir/*.zip 2>/dev/null | head -1` # whichever archive of yearly vrt files sits in workdir/; members are streamed, never extracted to disk
+if [[ -z "$ZIPFILE" ]] ; then
+ echo "Error: klk2eureco.sh: no .zip file found in $SCRIPTDIR/workdir" >&2
+ exit 1
+fi
XML="$CORPUS/XML" # output zipped vrtxml files of selection of newspapers
@@ -41,25 +46,23 @@
# Generate corpus files by year and source in vrt
-#for VRT in $DAT/*.vrt # loop to be applied to yearly files
-for VRT in "$DAT/klk_fi_v2_2021.vrt"
+for ZIPMEMBER in `unzip -Z1 "$ZIPFILE" | grep '\.vrt$'` # loop over every yearly vrt file found in the archive
do # (
- BASENAME=`basename $VRT .vrt`
+ BASENAME=`basename $ZIPMEMBER .vrt`
YEAR=`echo $BASENAME | gawk 'BEGIN {FS="_"} {print $4}'`
- ##TMP echo "Generating proper XML files from $VRT in $XML/$YEAR by source..."
- ##TMP gawk -v OUTDIR="$XML" -v YEAR="$YEAR" -f vrt2xml.awk $VRT
-
+ echo "Generating proper XML files from $ZIPMEMBER in $ZIPFILE, in $XML by source..."
+ unzip -p "$ZIPFILE" "$ZIPMEMBER" | gawk -v OUTDIR="$XML" -v YEAR="$YEAR" -f vrt2xml.awk
-
+
+
#--------------------------------------------------------
# Checking Wellformedness of the XML and generating TEI
#--------------------------------------------------------
- ##TMP for s in $XML/*/
- for s in $XML/Suomen_Kuvalehti/
- do # ( # threading
+ for s in $XML/*/ # every source vrt2xml.awk actually produced output for
+ do # ( # threading
SOURCE=`basename $s`
x=$s/$SOURCE$YEAR.xml