klk2eureco.sh

#/bin/bash

EURECO="/scratch/project_2010889/eureco"
TEST="/scratch/project_2010889/TEST"

CORPUS="$EURECO/klk-fi-v2-vrt"
TEIDTD="$EURECO/tei/tei_all.dtd"

DAT="$CORPUS/data"                # input vrt files of klk-fi by year
XML="$CORPUS/XML"                 # output zipped vrtxml files of selection of newspapers


TEI="$CORPUS/TEI"                 # output zipped tei files of the selection
#TEI="$TEST/TEI"                 # output zipped tei files of the selection

#TEIFORMAT="tei";
TEIFORMAT="i5";

MASK="";
#MASK="-m";

PERL_BAD=0;                      # to supress warnings about locale

mkdir -p $TEST
mkdir -p $XML
mkdir -p $TEI



# Multiple threads for a loop in bash:

## for stuff in things
## do
## ( something
##   with
##   stuff ) &
## done
## wait # for all the something with stuff



# Generate corpus files by year and source in vrt

# for VRT in $DAT/*.vrt                              # loop to be applied to yearly files
for VRT in $DAT/klk_fi_v2_201[0-3].vrt        
# for VRT in "$DAT/klk_fi_v2_2015.vrt"        
# for VRT in "$DAT/klk_fi_v2_2019.vrt" "$DAT/klk_fi_v2_2020.vrt" "$DAT/klk_fi_v2_2021.vrt"  
do # (

    BASENAME=`basename $VRT .vrt`
    YEAR=`echo $BASENAME | gawk 'BEGIN {FS="_"} {print $4}'`

    echo "Generating proper XML files from $VRT in $XML/$YEAR by source..."
    gawk -v OUTDIR="$XML" -v YEAR="$YEAR" -f vrt2xml.awk $VRT

    #--------------------------------------------------------
    # Checking Wellformedness of the XML and generating TEI
    #--------------------------------------------------------

    for s in $XML/*/
    # for s in $XML/Suomen_Kuvalehti/
    do ( # threading	
	SOURCE=`basename $s`
	x=$s/$SOURCE$YEAR.xml

	# Some checks of the XML beforehand:

	if [[ ! -f $x ]] ; then
	    echo "Warning: klk2eureco.sh: File $x does not exist; skipping"
	    break;
	fi

	if [[ -z `grep "<text " $x` ]] ; then
	    echo "Warning: klk2eureco.sh: File $x does not contain any <text>s ; skipping"
	    break;
	fi
	
	echo "  checking wellformedness of $x"
	xmllint --stream --noout $x 
	if [[ $R != "" ]] ; then
	    echo "Error: xmllint error with error return code $R" >&2;    
	    break; 
	fi

	
	#--------------------
	# calling vrt2tei.pl
	#--------------------
	
	BASENAME=`basename $x .xml`
	mkdir -p $TEI/$SOURCE
	t="$TEI/$SOURCE/$BASENAME.$TEIFORMAT.xml"
	
	echo "  generating $t using vrt2tei.pl, and prettifying..."
	perl vrt2tei.pl -t $TEIFORMAT $MASK $x |
	    xml_pp |                           # xml_pp works but takes ages;
	    gawk -f rearrange-idsDoc.awk > $t  # rearranging the start and end tag of idsDoc
	                                       # xmllint- format geht out of memory und --stream machen funktioniert nicht
	ls -l $t

	if [ "$TEIFORMAT" == "i5" ] ; then
	    echo "  validating $t..."
	    xmllint --stream --noout --valid $t
	else
	    echo "  validating $t..."
	    xmllint --stream --noout --dtdvalid $TEIDTD $t         # scheint so zu funktioneren - nicht kombinieren mit --format!
	fi
	
	# echo "  zipping $x..."
	# gzip -f $x
	
	# echo "  zipping $t..."
	# gzip -f $t
	
	echo ) & 
    done
    wait # ) &                                    # wait seems to have to be there for it to terminate 
   
done
# wait