use publ_id
diff --git a/klk2eureco.sh b/klk2eureco.sh
index a01e077..7fe0386 100755
--- a/klk2eureco.sh
+++ b/klk2eureco.sh
@@ -13,8 +13,8 @@
TEI="$CORPUS/TEI" # output zipped tei files of the selection
#TEI="$TEST/TEI" # output zipped tei files of the selection
-#TEIFORMAT="tei";
-TEIFORMAT="i5";
+TEIFORMAT="tei";
+#TEIFORMAT="i5";
MASK="";
#MASK="-m";
@@ -41,48 +41,57 @@
# Generate corpus files by year and source in vrt
-# for VRT in $DAT/*.vrt # loop to be applied to yearly files
-for VRT in $DAT/klk_fi_v2_201[0-3].vrt
-# for VRT in "$DAT/klk_fi_v2_2015.vrt"
-# for VRT in "$DAT/klk_fi_v2_2019.vrt" "$DAT/klk_fi_v2_2020.vrt" "$DAT/klk_fi_v2_2021.vrt"
+#for VRT in $DAT/*.vrt # loop to be applied to yearly files
+for VRT in "$DAT/klk_fi_v2_2021.vrt"
do # (
BASENAME=`basename $VRT .vrt`
YEAR=`echo $BASENAME | gawk 'BEGIN {FS="_"} {print $4}'`
- echo "Generating proper XML files from $VRT in $XML/$YEAR by source..."
- gawk -v OUTDIR="$XML" -v YEAR="$YEAR" -f vrt2xml.awk $VRT
+ ##TMP echo "Generating proper XML files from $VRT in $XML/$YEAR by source..."
+ ##TMP gawk -v OUTDIR="$XML" -v YEAR="$YEAR" -f vrt2xml.awk $VRT
+
+
#--------------------------------------------------------
# Checking Wellformedness of the XML and generating TEI
#--------------------------------------------------------
- for s in $XML/*/
- # for s in $XML/Suomen_Kuvalehti/
- do ( # threading
+ ##TMP for s in $XML/*/
+ for s in $XML/Suomen_Kuvalehti/
+ do # ( # threading
SOURCE=`basename $s`
x=$s/$SOURCE$YEAR.xml
- # Some checks of the XML beforehand:
+ #---------------------------------------------------------------
+ # Some existence checks and removal of empty files beforehand:
+ #---------------------------------------------------------------
+
+ if [[ ! -f $x ]] ; then
+ echo "Warning: klk2eureco.sh: File $x does not exist; skipping";
+ break;
+ fi
+
+ if [[ -z `grep -l "<text " $x` ]] ; then
+ #echo "Warning: klk2eureco.sh: File $x does not contain any <text>s, skipping"
+ echo "Warning: klk2eureco.sh: File $x does not contain any <text>s,removing"
+ rm -fv $x;
+ break;
+ fi
- if [[ ! -f $x ]] ; then
- echo "Warning: klk2eureco.sh: File $x does not exist; skipping"
- break;
- fi
-
- if [[ -z `grep "<text " $x` ]] ; then
- echo "Warning: klk2eureco.sh: File $x does not contain any <text>s ; skipping"
- break;
- fi
-
+ #---------------------------------
+ # checking well-formedness of $x
+ #---------------------------------
+
echo " checking wellformedness of $x"
- xmllint --stream --noout $x
+ xmllint --stream --noout $x
if [[ $R != "" ]] ; then
echo "Error: xmllint error with error return code $R" >&2;
break;
fi
+
#--------------------
# calling vrt2tei.pl
#--------------------
@@ -98,6 +107,12 @@
# xmllint- format geht out of memory und --stream machen funktioniert nicht
ls -l $t
+ # if $t is empty, remove it:
+ if [ ! -s $i ]; then
+ rm -f $i
+ fi
+
+ # validating $t:
if [ "$TEIFORMAT" == "i5" ] ; then
echo " validating $t..."
xmllint --stream --noout --valid $t
@@ -106,13 +121,13 @@
xmllint --stream --noout --dtdvalid $TEIDTD $t # scheint so zu funktioneren - nicht kombinieren mit --format!
fi
- # echo " zipping $x..."
- # gzip -f $x
-
- # echo " zipping $t..."
- # gzip -f $t
-
- echo ) &
+###TMP # echo " zipping $x..."
+###TMP # gzip -f $x
+###TMP
+###TMP # echo " zipping $t..."
+###TMP # gzip -f $t
+
+ echo # ) &
done
wait # ) & # wait seems to have to be there for it to terminate
diff --git a/vrt2tei.pl b/vrt2tei.pl
index 3add1e7..86abb6f 100755
--- a/vrt2tei.pl
+++ b/vrt2tei.pl
@@ -243,23 +243,32 @@
# (ToDo: the following hashes could probably conflated into an array of hashes or so)
- # set full titles ($flarray[4]) as keys:
- $corpusids{$flarray[4]} = $flarray[2];
- $srcpublids{$flarray[4]} = $flarray[3];
- $srcfullnames{$flarray[4]} = $flarray[4];
- $srcpubplaces{$flarray[4]} = $flarray[9];
- $srcpublishers{$flarray[4]} = $flarray[10];
- $srctexttypes{$flarray[4]} = $flarray[7];
- $srctextlangs{$flarray[4]} = $flarray[8];
+ # set full titles ($flarray[3]) as keys:
+ $corpusids{$flarray[3]} = $flarray[1];
+ $srcpublids{$flarray[3]} = $flarray[2];
+ $srcfullnames{$flarray[3]} = $flarray[3];
+ $srcpubplaces{$flarray[3]} = $flarray[8];
+ $srcpublishers{$flarray[3]} = $flarray[9];
+ $srctexttypes{$flarray[3]} = $flarray[6];
+ $srctextlangs{$flarray[3]} = $flarray[7];
# also set simple titles ($flarray[5]) as keys:
- $corpusids{$flarray[5]} = $flarray[2];
- $srcpublids{$flarray[5]} = $flarray[3];
- $srcfullnames{$flarray[5]} = $flarray[4];
- $srcpubplaces{$flarray[5]} = $flarray[9];
- $srcpublishers{$flarray[5]} = $flarray[10];
- $srctexttypes{$flarray[5]} = $flarray[7];
- $srctextlangs{$flarray[5]} = $flarray[8];
+ $corpusids{$flarray[4]} = $flarray[1];
+ $srcpublids{$flarray[4]} = $flarray[2];
+ $srcfullnames{$flarray[4]} = $flarray[3];
+ $srcpubplaces{$flarray[4]} = $flarray[8];
+ $srcpublishers{$flarray[4]} = $flarray[9];
+ $srctexttypes{$flarray[4]} = $flarray[6];
+ $srctextlangs{$flarray[4]} = $flarray[7];
+
+ # also set publids ($flarray[2]) as keys:
+ $corpusids{$flarray[2]} = $flarray[1];
+ $srcpublids{$flarray[2]} = $flarray[2];
+ $srcfullnames{$flarray[2]} = $flarray[3];
+ $srcpubplaces{$flarray[2]} = $flarray[8];
+ $srcpublishers{$flarray[2]} = $flarray[9];
+ $srctexttypes{$flarray[2]} = $flarray[6];
+ $srctextlangs{$flarray[2]} = $flarray[7];
}
close($SOURCES);
@@ -591,7 +600,7 @@
# USE 17 page_id="p1"
# USE 18 page_no="None"
# 19 part_name="_"
- # 20 publ_id="0039-5552"
+ # USE 20 publ_id="0039-5552"
# 21 publ_part=""
# USE 22 publ_title="Suomen Kuvalehti"
# USE 23 publ_type="aikakausi"
@@ -616,6 +625,7 @@
my $PAGEID = $textattsref->{'page_id'};
my $PAGENO = $textattsref->{'page_no'};
my $PUBLTITLE = $textattsref->{'publ_title'};
+ my $PUBLID = $textattsref->{'publ_id'};
my $PUBLTYPE = $textattsref->{'publ_type'};
my $SENTCOUNT = $textattsref->{'sentcount'};
my $SUMLANG = $textattsref->{'sum_lang'};
@@ -733,9 +743,9 @@
$monogr->get_xpath('./imprint/' . $date . '[@type="year"]', 0) ->set_text($datearray[0]);
$monogr->get_xpath('./imprint/' . $date . '[@type="month"]', 0) ->set_text($datearray[1]);
$monogr->get_xpath('./imprint/' . $date . '[@type="day"]', 0) ->set_text($datearray[2]);
- $monogr->first_child("imprint")->first_child("pubPlace") ->set_text($srcpubplaces{$PUBLTITLE}); # imprint is needed for tei validity
+ $monogr->first_child("imprint")->first_child("pubPlace") ->set_text($srcpubplaces{$PUBLID}); # imprint is needed for tei validity
$monogr->first_child("imprint")->first_child("pubPlace") ->set_att('key', $CountryKey);
- $monogr->first_child("imprint")->first_child("publisher") ->set_text($srcpublishers{$PUBLTITLE}); # imprint is needed for tei validity
+ $monogr->first_child("imprint")->first_child("publisher") ->set_text($srcpublishers{$PUBLID}); # imprint is needed for tei validity
#$monogr->get_xpath('./biblScope[@unit="ISSUETITLE"]', 0) ->set_text($ISSUETITLE);
#$monogr->get_xpath('./biblScope[@unit="ISSUENO"]', 0) ->set_text($ISSUENO);
#$monogr->get_xpath('./biblScope[@unit="ISSUEDATE"]', 0) ->set_text($ISSUEDATE);
@@ -940,14 +950,15 @@
sub set_sourceDescI5{
my ($corpusHeader) = @_;
+ my $PUBLID = $srcpublids{$fnsource};
my $PUBLTITLE = $srcfullnames{$fnsource};
- my $PUBLPLACE = $srcpubplaces{$PUBLTITLE};
- my $PUBLISHER = $srcpublishers{$PUBLTITLE};
+ my $PUBLPLACE = $srcpubplaces{$PUBLID};
+ my $PUBLISHER = $srcpublishers{$PUBLID};
my $YEAR = $fnyear;
my $YY = substr($fnyear, 2, 2);
- my $CSIGLE = $corpusids{$PUBLTITLE} . $YY;
+ my $CSIGLE = $corpusids{$PUBLID} . $YY;
#<idsHeader>
@@ -985,13 +996,15 @@
my ($textattsref) = @_;
my $DATE = $textattsref->{'date'};
my $PUBLTITLE = $textattsref->{'publ_title'};
+ my $PUBLID = $textattsref->{'publ_id'};
+
my @datearray = split("-", $DATE);
my $MONTH = $datearray[1];
my $YEAR = $datearray[0];
my $YY = substr($YEAR, 2, 2);
- my $CSIGLE = $corpusids{$PUBLTITLE} . $YY;
+ my $CSIGLE = $corpusids{$PUBLID} . $YY;
my $DOCID = $months{$MONTH};
my $MONTHNAME = $monthnames{$MONTH};