klk2eureco.sh

#/bin/bash

EURECO="/scratch/project_2010889/eureco"
TEST="/scratch/project_2010889/TEST"

CORPUS="$EURECO/klk-fi-v2-vrt"
TEIDTD="$EURECO/tei/tei_all.dtd"

DAT="$CORPUS/data"                # input vrt files of klk-fi by year
XML="$CORPUS/XML"                 # output zipped vrtxml files of selection of newspapers


TEI="$CORPUS/TEI"                 # output zipped tei files of the selection
#TEI="$TEST/TEI"                 # output zipped tei files of the selection

TEIFORMAT="tei";
#TEIFORMAT="i5";

MASK="";
#MASK="-m";

PERL_BAD=0;                      # to supress warnings about locale

mkdir -p $TEST
mkdir -p $XML
mkdir -p $TEI



# Multiple threads for a loop in bash:

## for stuff in things
## do
## ( something
##   with
##   stuff ) &
## done
## wait # for all the something with stuff



# Generate corpus files by year and source in vrt

#for VRT in $DAT/*.vrt                              # loop to be applied to yearly files
for VRT in "$DAT/klk_fi_v2_2021.vrt"        
do # (

    BASENAME=`basename $VRT .vrt`
    YEAR=`echo $BASENAME | gawk 'BEGIN {FS="_"} {print $4}'`

    ##TMP echo "Generating proper XML files from $VRT in $XML/$YEAR by source..."
    ##TMP gawk -v OUTDIR="$XML" -v YEAR="$YEAR" -f vrt2xml.awk $VRT
    

    
    #--------------------------------------------------------
    # Checking Wellformedness of the XML and generating TEI
    #--------------------------------------------------------

    ##TMP for s in $XML/*/
    for s in $XML/Suomen_Kuvalehti/
    do # ( # threading	
	SOURCE=`basename $s`
	x=$s/$SOURCE$YEAR.xml

	#---------------------------------------------------------------
 	# Some existence checks and removal of empty files beforehand:
        #---------------------------------------------------------------
 	
 	if [[ ! -f $x ]] ; then
 	    echo "Warning: klk2eureco.sh: File $x does not exist; skipping";
 	    break;
 	fi
 
 	if [[ -z `grep -l "<text " $x` ]] ; then
 	    #echo "Warning: klk2eureco.sh: File $x does not contain any <text>s, skipping"
 	    echo "Warning: klk2eureco.sh: File $x does not contain any <text>s,removing"
   	    rm -fv $x;
 	    break;
 	fi

	#---------------------------------
	# checking well-formedness of $x
	#---------------------------------
 	
	echo "  checking wellformedness of $x"
	xmllint --stream --noout $x
	if [[ $R != "" ]] ; then
	    echo "Error: xmllint error with error return code $R" >&2;    
	    break; 
	fi

	
	
	#--------------------
	# calling vrt2tei.pl
	#--------------------
	
	BASENAME=`basename $x .xml`
	mkdir -p $TEI/$SOURCE
	t="$TEI/$SOURCE/$BASENAME.$TEIFORMAT.xml"
	
	echo "  generating $t using vrt2tei.pl, and prettifying..."
	perl vrt2tei.pl -t $TEIFORMAT $MASK $x |
	    xml_pp |                           # xml_pp works but takes ages;
	    gawk -f rearrange-idsDoc.awk > $t  # rearranging the start and end tag of idsDoc
	                                       # xmllint- format geht out of memory und --stream machen funktioniert nicht
	ls -l $t

	# if $t is empty, remove it:
	if [ ! -s $i ]; then
	    rm -f $i
	fi

	# validating $t:
	if [ "$TEIFORMAT" == "i5" ] ; then
	    echo "  validating $t..."
	    xmllint --stream --noout --valid $t
	else
	    echo "  validating $t..."
	    xmllint --stream --noout --dtdvalid $TEIDTD $t         # scheint so zu funktioneren - nicht kombinieren mit --format!
	fi
	
###TMP 	# echo "  zipping $x..."
###TMP 	# gzip -f $x
###TMP 	
###TMP 	# echo "  zipping $t..."
###TMP 	# gzip -f $t

 	echo # ) & 
    done
    wait # ) &                                    # wait seems to have to be there for it to terminate 
   
done
# wait