klk2eureco.sh

#/bin/bash

EURECO="/scratch/project_2010889/eureco"
TEST="/scratch/project_2010889/TEST"

CORPUS="$EURECO/klk-fi-v2-vrt"
TEIDTD="$EURECO/tei/tei_all.dtd"

DAT="$CORPUS/data"                # input vrt files of klk-fi by year
XML="$CORPUS/XML"                 # output zipped vrtxml files of selection of newspapers
TEI="$CORPUS/TEI"                 # output zipped tei files of the selection
#TEI="$TEST/TEI"                 # output zipped tei files of the selection

#TEIFORMAT="tei";
TEIFORMAT="i5";

mkdir -p $TEST
mkdir -p $XML
mkdir -p $TEI



# Multiple threads for a loop in bash:

## for stuff in things
## do
## ( something
##   with
##   stuff ) &
## done
## wait # for all the something with stuff



# Generate corpus files by year and source in vrt

# for VRT in "$DAT/*.vrt"
#for VRT in "$DAT/klk_fi_v2_2021.vrt"         # to be applied to yearly files
# for VRT in "$DAT/klk_fi_v2_2020.vrt"        
for VRT in "$DAT/klk_fi_v2_2018.vrt" "$DAT/klk_fi_v2_2019.vrt"  
do # (

    BASENAME=`basename $VRT .vrt`
    YY=`echo $BASENAME | gawk 'BEGIN {FS="_"} {print $4}'`
    
    echo "Generating proper XML files from $VRT in $XML/$YY by source..."
    gawk -v OUTDIR="$XML" -v YEAR="$YY" -f vrt2xml.awk $VRT               # will generate corpus files for different sources in the YEAR dir 

done
exit

    
    # Checking Wellformedness of the XML and generating TEI
    for s in $XML/*/
    # for s in $XML/Suomen_Kuvalehti/
    # for s in $XML/Lansi-Savo/
    # for s in $XML/Helsingin_Sanomat/
    # for s in $XML/Aamulehti/ $XML/Etela-Suomen_Sanomat/ $XML/Hameen_Sanomat/ $XML/Helsingin_Sanomat/ $XML/Ilkka-Pohjalainen/ $XML/Ilta-Sanomat/ $XML/Kaleva/ $XML/Keskipohjanmaa/ $XML/Satakunnan_Kansa/ $XML/Savon_Sanomat/ $XML/Turun_Sanomat/
    do ( # threading	
	SOURCE=`basename $s`
	x=$s/$SOURCE$YY.xml
	
 	## echo "  checking wellformedness of $x"
	## xmllint --noout $x                                                # ToDo: make if condition for continuing only if well-formed
	# if($R != 0){
	#      echo "Error: xmllint error with error return code $R" >&2;    
	#      break; 
	#  }
	      
	BASENAME=`basename $x .xml`
	mkdir -p $TEI/$SOURCE
	t="$TEI/$SOURCE/$BASENAME.$TEIFORMAT.xml"
	
	echo "  generating $t using vrt2tei.pl, and prettifying..."
	./vrt2tei.pl $x |
	    xml_pp |                           # xml_pp works but takes ages;
	    gawk -f rearrange-idsDoc.awk > $t  # rearranging the start and end tag of idsDoc
	                                       # xmllint- format geht out of memory und --stream machen funktioniert nicht
	ls -l $t

	echo "  validating..."
	if [ "$TEIFORMAT" == "i5" ] ; then
	    xmllint --stream --noout --valid $t
	else 
	    xmllint --stream --noout --dtdvalid $TEIDTD $t              # TEI; scheint so zu funktioneren - nicht kombinieren mit --format!
	fi
	
	# echo "  zipping $x..."
	# gzip -f $x
	
	# echo "  zipping $t..."
	# gzip -f $t
	
	echo ) & 
    done
    # wait # ) &                                                    # wait does not seem to make sense if nothing follows
   
done
# wait