klk2eureco.sh

#/bin/bash

EURECO="/scratch/project_2010889/eureco"
TEST="/scratch/project_2010889/TEST"

CORPUS="$EURECO/klk-fi-v2-vrt"
TEIDTD="$EURECO/tei/tei_all.dtd"

SCRIPTDIR="$(cd "$(dirname "$0")" && pwd)"
ZIPFILE=`ls "$SCRIPTDIR"/workdir/*.zip 2>/dev/null | head -1`   # whichever archive of yearly vrt files sits in workdir/; members are streamed, never extracted to disk
if [[ -z "$ZIPFILE" ]] ; then
    echo "Error: klk2eureco.sh: no .zip file found in $SCRIPTDIR/workdir" >&2
    exit 1
fi
XML="$CORPUS/XML"                 # output zipped vrtxml files of selection of newspapers


TEI="$CORPUS/TEI"                 # output zipped tei files of the selection
#TEI="$TEST/TEI"                 # output zipped tei files of the selection

TEIFORMAT="tei";
#TEIFORMAT="i5";

MASK="";
#MASK="-m";

PERL_BAD=0;                      # to supress warnings about locale

mkdir -p $TEST
mkdir -p $XML
mkdir -p $TEI



# Multiple threads for a loop in bash:

## for stuff in things
## do
## ( something
##   with
##   stuff ) &
## done
## wait # for all the something with stuff



# Generate corpus files by year and source in vrt

for ZIPMEMBER in `unzip -Z1 "$ZIPFILE" | grep '\.vrt$'`      # loop over every yearly vrt file found in the archive
do # (

    BASENAME=`basename $ZIPMEMBER .vrt`
    YEAR=`echo $BASENAME | gawk 'BEGIN {FS="_"} {print $4}'`

    echo "Generating proper XML files from $ZIPMEMBER in $ZIPFILE, in $XML by source..."
    unzip -p "$ZIPFILE" "$ZIPMEMBER" | gawk -v OUTDIR="$XML" -v YEAR="$YEAR" -f vrt2xml.awk



    #--------------------------------------------------------
    # Checking Wellformedness of the XML and generating TEI
    #--------------------------------------------------------

    for s in $XML/*/                                          # every source vrt2xml.awk actually produced output for
    do # ( # threading
	SOURCE=`basename $s`
	x=$s/$SOURCE$YEAR.xml

	#---------------------------------------------------------------
 	# Some existence checks and removal of empty files beforehand:
        #---------------------------------------------------------------
 	
 	if [[ ! -f $x ]] ; then
 	    echo "Warning: klk2eureco.sh: File $x does not exist; skipping";
 	    break;
 	fi
 
 	if [[ -z `grep -l "<text " $x` ]] ; then
 	    #echo "Warning: klk2eureco.sh: File $x does not contain any <text>s, skipping"
 	    echo "Warning: klk2eureco.sh: File $x does not contain any <text>s,removing"
   	    rm -fv $x;
 	    break;
 	fi

	#---------------------------------
	# checking well-formedness of $x
	#---------------------------------
 	
	echo "  checking wellformedness of $x"
	xmllint --stream --noout $x
	if [[ $R != "" ]] ; then
	    echo "Error: xmllint error with error return code $R" >&2;    
	    break; 
	fi

	
	
	#--------------------
	# calling vrt2tei.pl
	#--------------------
	
	BASENAME=`basename $x .xml`
	mkdir -p $TEI/$SOURCE
	t="$TEI/$SOURCE/$BASENAME.$TEIFORMAT.xml"
	
	echo "  generating $t using vrt2tei.pl..."
	perl vrt2tei.pl -t $TEIFORMAT $MASK $x |
	    gawk -f rearrange-idsDoc.awk > $t  # rearranging the start and end tag of idsDoc
	                                       # pretty-printing now done in-process by XML::Twig (pretty_print => 'record'),
	                                       # replacing the old xml_pp re-parse pass (xmllint --format runs out of memory, --stream can't format)
	ls -l $t

	# if $t is empty, remove it:
	if [ ! -s $t ]; then
	    rm -f $t
	fi

	# validating $t:
	if [ "$TEIFORMAT" == "i5" ] ; then
	    echo "  validating $t..."
	    xmllint --stream --noout --valid $t
	else
	    echo "  validating $t..."
	    xmllint --stream --noout --dtdvalid $TEIDTD $t         # scheint so zu funktioneren - nicht kombinieren mit --format!
	fi
	
###TMP 	# echo "  zipping $x..."
###TMP 	# gzip -f $x
###TMP 	
###TMP 	# echo "  zipping $t..."
###TMP 	# gzip -f $t

 	echo # ) & 
    done
    wait # ) &                                    # wait seems to have to be there for it to terminate 
   
done
# wait