checks of the XML files
diff --git a/klk2eureco.sh b/klk2eureco.sh
index 027f06b..a01e077 100755
--- a/klk2eureco.sh
+++ b/klk2eureco.sh
@@ -8,12 +8,19 @@
DAT="$CORPUS/data" # input vrt files of klk-fi by year
XML="$CORPUS/XML" # output zipped vrtxml files of selection of newspapers
+
+
TEI="$CORPUS/TEI" # output zipped tei files of the selection
#TEI="$TEST/TEI" # output zipped tei files of the selection
#TEIFORMAT="tei";
TEIFORMAT="i5";
+MASK="";
+#MASK="-m";
+
+PERL_BAD=0; # to supress warnings about locale
+
mkdir -p $TEST
mkdir -p $XML
mkdir -p $TEI
@@ -34,55 +41,69 @@
# Generate corpus files by year and source in vrt
-# for VRT in "$DAT/*.vrt"
-#for VRT in "$DAT/klk_fi_v2_2021.vrt" # to be applied to yearly files
-# for VRT in "$DAT/klk_fi_v2_2020.vrt"
-for VRT in "$DAT/klk_fi_v2_2018.vrt" "$DAT/klk_fi_v2_2019.vrt"
+# for VRT in $DAT/*.vrt # loop to be applied to yearly files
+for VRT in $DAT/klk_fi_v2_201[0-3].vrt
+# for VRT in "$DAT/klk_fi_v2_2015.vrt"
+# for VRT in "$DAT/klk_fi_v2_2019.vrt" "$DAT/klk_fi_v2_2020.vrt" "$DAT/klk_fi_v2_2021.vrt"
do # (
BASENAME=`basename $VRT .vrt`
- YY=`echo $BASENAME | gawk 'BEGIN {FS="_"} {print $4}'`
-
- echo "Generating proper XML files from $VRT in $XML/$YY by source..."
- gawk -v OUTDIR="$XML" -v YEAR="$YY" -f vrt2xml.awk $VRT # will generate corpus files for different sources in the YEAR dir
+ YEAR=`echo $BASENAME | gawk 'BEGIN {FS="_"} {print $4}'`
-done
-exit
+ echo "Generating proper XML files from $VRT in $XML/$YEAR by source..."
+ gawk -v OUTDIR="$XML" -v YEAR="$YEAR" -f vrt2xml.awk $VRT
-
+ #--------------------------------------------------------
# Checking Wellformedness of the XML and generating TEI
+ #--------------------------------------------------------
+
for s in $XML/*/
# for s in $XML/Suomen_Kuvalehti/
- # for s in $XML/Lansi-Savo/
- # for s in $XML/Helsingin_Sanomat/
- # for s in $XML/Aamulehti/ $XML/Etela-Suomen_Sanomat/ $XML/Hameen_Sanomat/ $XML/Helsingin_Sanomat/ $XML/Ilkka-Pohjalainen/ $XML/Ilta-Sanomat/ $XML/Kaleva/ $XML/Keskipohjanmaa/ $XML/Satakunnan_Kansa/ $XML/Savon_Sanomat/ $XML/Turun_Sanomat/
do ( # threading
SOURCE=`basename $s`
- x=$s/$SOURCE$YY.xml
+ x=$s/$SOURCE$YEAR.xml
+
+ # Some checks of the XML beforehand:
+
+ if [[ ! -f $x ]] ; then
+ echo "Warning: klk2eureco.sh: File $x does not exist; skipping"
+ break;
+ fi
+
+ if [[ -z `grep "<text " $x` ]] ; then
+ echo "Warning: klk2eureco.sh: File $x does not contain any <text>s ; skipping"
+ break;
+ fi
- ## echo " checking wellformedness of $x"
- ## xmllint --noout $x # ToDo: make if condition for continuing only if well-formed
- # if($R != 0){
- # echo "Error: xmllint error with error return code $R" >&2;
- # break;
- # }
-
+ echo " checking wellformedness of $x"
+ xmllint --stream --noout $x
+ if [[ $R != "" ]] ; then
+ echo "Error: xmllint error with error return code $R" >&2;
+ break;
+ fi
+
+
+ #--------------------
+ # calling vrt2tei.pl
+ #--------------------
+
BASENAME=`basename $x .xml`
mkdir -p $TEI/$SOURCE
t="$TEI/$SOURCE/$BASENAME.$TEIFORMAT.xml"
echo " generating $t using vrt2tei.pl, and prettifying..."
- ./vrt2tei.pl $x |
+ perl vrt2tei.pl -t $TEIFORMAT $MASK $x |
xml_pp | # xml_pp works but takes ages;
gawk -f rearrange-idsDoc.awk > $t # rearranging the start and end tag of idsDoc
# xmllint- format geht out of memory und --stream machen funktioniert nicht
ls -l $t
- echo " validating..."
if [ "$TEIFORMAT" == "i5" ] ; then
+ echo " validating $t..."
xmllint --stream --noout --valid $t
- else
- xmllint --stream --noout --dtdvalid $TEIDTD $t # TEI; scheint so zu funktioneren - nicht kombinieren mit --format!
+ else
+ echo " validating $t..."
+ xmllint --stream --noout --dtdvalid $TEIDTD $t # scheint so zu funktioneren - nicht kombinieren mit --format!
fi
# echo " zipping $x..."
@@ -93,7 +114,7 @@
echo ) &
done
- # wait # ) & # wait does not seem to make sense if nothing follows
+ wait # ) & # wait seems to have to be there for it to terminate
done
# wait