blob: 54b4c8d675ba63996bbc3579937658ff348507ca [file] [log] [blame]
Harald Lüngen9d4e0462024-08-23 09:34:22 +03001#! /usr/bin/perl -w
2
3
4###########################################################################################################################################################
5# vrt2tei.pl
6# eureco
7# leibniz-institut fuer deutsche sprache / csc finland esbo
8# august 2024
9#
10#
11# using XML::Twig , see http://www.xmltwig.org/, https://metacpan.org/pod/XML::Twiga
12#
Harald Lüngenccd84902024-08-27 16:03:47 +030013# usage: see below the usage fugnction
Harald Lüngencaab0802024-08-23 17:28:22 +030014# Usage: ./vrt2tei.pl <vrtxmlfile.xml> <outfile>
Harald Lüngen9d4e0462024-08-23 09:34:22 +030015# <vrtxmlfile>: xml-ised vrt file
16#
17#
18# TODO:
19# 1 insert dtd spec, or ref to TEI
20
Harald Lüngendb5e6e72024-09-04 17:41:18 +030021# 3a remove the vrt positional attribute comment line / all comment lines
Harald Lüngen9d4e0462024-08-23 09:34:22 +030022# 3b add @head and @deprel to I5 sowie auch @msd
23# 3c bearbeitung von @head und @deprel in tei2korapxml durch Nils?
24# 3d build 30 billion corpus
25
26# 4a take care of IDs
27# 4b see to the values of @xml:lang
28# 5 abfangen von unerwarteten elementen dh andere als <sentence> und <paragraph>
29# 5a wort reihenfolge nochmal checken
30# 6 checks and balances
Harald Lüngen9d4e0462024-08-23 09:34:22 +030031# 7 How to encode Kielipankki and National Library of Finland? in teiCorpus Header
32# 8 construct <idsDoc>s for the months (or go for TEI)
33# 9 parallelisation in bash and application on sub corpora of KLK
34# 10 re-implementation of the gawk code in the perl script
35# 12 re-implement creation of text header from xml file in another twig / parametrize TEI vs I5
36
37
38
39#remember
40#formatted.xml:105613: element w: validity error : No declaration for attribute deprel of element w
41#formatted.xml:105613: element w: validity error : No declaration for attribute head of element w
42
43
44#
45#
46############################################################################################################################################################
47
48
49use strict;
50use warnings;
51
52use XML::Twig;
53use XML::Generator ':pretty'; # apparently no effect when using flush();
54
55
56use locale; # diese drei Zeilen, damit \b im regex nicht Umlaute und ß matcht.
57use POSIX qw(locale_h); # to be able to use setlocale()
58#setlocale(LC_ALL,'de_DE');
59setlocale(LC_ALL, "fi_FI");
60use utf8;
61use open qw( :std :encoding(UTF-8) );
62
63use Time::Piece;
64use Tie::IxHash;
65
Harald Lüngendb5e6e72024-09-04 17:41:18 +030066
67
Harald Lüngen9d4e0462024-08-23 09:34:22 +030068#----------------------
69# check file arguments:
70#----------------------
71
72# arg0 infile: vrt-xml
Harald Lüngen9d4e0462024-08-23 09:34:22 +030073
Harald Lüngena20e69d2024-08-29 13:33:08 +030074unless($ARGV[0]) {&usage_message()} ; # min arg0, the input file
75if ($ARGV[1]) {&usage_message()}; # max arg0, the input file
Harald Lüngen9d4e0462024-08-23 09:34:22 +030076
77
78####################
79# GLOBAL VARIABLES
80####################
81
82my $encoding = "UTF-8";
83#my $encoding = "iso-8859-1"; # dieses $encoding ist NUR fuer das output s.u. twig funktion
Harald Lüngencaab0802024-08-23 17:28:22 +030084my $textcounter = 0;
Harald Lüngen9d4e0462024-08-23 09:34:22 +030085
Harald Lüngen86cbd932024-09-10 15:52:18 +030086my %doccounter = ( # by the month as in derekox
87 "01" => 1,
88 "02" => 1,
89 "03" => 1,
90 "04" => 1,
91 "05" => 1,
92 "06" => 1,
93 "07" => 1,
94 "08" => 1,
95 "09" => 1,
96 "10" => 1,
97 "11" => 1,
98 "12" => 1,
99 );
100
101
102my $sourcescsvfile = "sources_klk_fi_v2_2021_4eureco.csv";
103my $corpheaderfile = "teiCorpusHeaderSkeleton.tei.xml";
104my $textheaderfile = "teiTextHeaderSkeleton.tei.xml";
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300105
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300106
107my $twig="";
108my $teiCorpusHeaderDoc="";
109
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300110# global variables pertaining to the original corpus :
111my $kielipankkiCorpus = "klk-fi-v2-vrt";
112
Harald Lüngen86cbd932024-09-10 15:52:18 +0300113my %months = (
114 "01" => "JAN",
115 "02" => "FEB",
116 "03" => "MAR",
117 "04" => "APR",
118 "05" => "MAY",
119 "06" => "JUN",
120 "07" => "JUL",
121 "08" => "AUG",
122 "09" => "SEP",
123 "10" => "OCT",
124 "11" => "NOV",
125 "12" => "DEC",
126 );
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300127
128#------------------------------------------------------------------
129# read corpusHeaderSkeleton document and get header out of it
130#------------------------------------------------------------------
131
132my $teiCorpusHeaderDocTwig = new XML::Twig(
133 keep_spaces => 1,
134 keep_atts_order => 1,
135 comments => 'drop',
136 );
137
138
Harald Lüngen86cbd932024-09-10 15:52:18 +0300139$teiCorpusHeaderDocTwig->parsefile($corpheaderfile);
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300140my $corpusHeader = $teiCorpusHeaderDocTwig->root; # getting the teiHeader for corpus out of the teiCorpusHeaderSkeleton document
141
142
143#------------------------------------------------------------------
144# read textHeaderSkeleton document adn get header out of it
145#------------------------------------------------------------------
146
147my $teiTextHeaderDocTwig = new XML::Twig(
148 keep_spaces => 1,
149 keep_atts_order => 1,
150 comments => 'drop',
151 );
152
Harald Lüngen86cbd932024-09-10 15:52:18 +0300153$teiTextHeaderDocTwig->parsefile($textheaderfile);
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300154my $textHeader = $teiTextHeaderDocTwig->root; # getting the teiHeader for corpus out of the teiTextHeaderSkeleton document
155
156
157#----------------------------------
158# read input VRT-XML document
159#----------------------------------
160
161open(my $IN, "< $ARGV[0]") || die("$0: cannot open file for reading: $ARGV[0]"); # open input file and initialise filehandel, actually does not seem to be needed
162 # as parsefile() (s.b.) is applied to the filename
163
Harald Lüngen86cbd932024-09-10 15:52:18 +0300164#-------------------------------------------------------------------------------------------
165# read source metadata file (prepared manually => ultimately read the info from CMDI File?)
166#-------------------------------------------------------------------------------------------
167
168my $linecount=0;
169open(my $SOURCES, $sourcescsvfile) || die("$0: cannot open file for reading: $sourcescsvfile");
170while(my $line = $SOURCES){
171 $linecount++;
172 chomp($line);
173 if ( $line=~/^\#/ || $line=~/^\s*$/ || $linecount == 1){ # skip line if emmpty line or comment line or first line
174 next;
175 };
176 # split each line into array:
177 my @line = split(/\t+/, $line);
178
179 # $sources{"Suomen Kuvalehti"}[0]
180
181}
182
183# while (my $line = <$seq_fh>) {
184# chomp $line;
185# ## skip comments and blank lines and optional repeat of title line
186# next if $line =~ /^\#/ || $line =~ /^\s*$/ || $line =~ /^\+/;
187# #split each line into array
188# my @line = split(/\s+/, $line);
189# $result{$line[0]}{yeartotal} += $line[1];
190# $result{$line[0]}{earning} += $line[3] - $line[2];
191# }
192
193
194
195
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300196
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300197#####################
198# M A I N
199#####################
200
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300201#-------------------------------------------------------------------------------------------------------------
202# start twig for input and call start tag handler for root and twig handler for each <text> in the VRT
203#-------------------------------------------------------------------------------------------------------------
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300204
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300205
206$twig = new XML::Twig(
207 keep_spaces => 1, # dadurch auch whitespaces an ehemeligen elementgrenzen im output
208 keep_atts_order => 1, # requires Tie::IxHash
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300209 comments => 'drop',
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300210 start_tag_handlers => {
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300211 texts => sub{root(@_, $corpusHeader)}
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300212 },
213 twig_handlers => {
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300214# text => \&text
Harald Lüngen7abb0b52024-09-05 16:26:57 +0300215 text => sub{text(@_, $textHeader->copy)} # copy must be because textHeader will be flushed with $twig in the <text> handler;
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300216 },
217 # dtd_handlers => { # ToDo for I5
218 # \&set_dtd;
219 # }
Harald Lüngena20e69d2024-08-29 13:33:08 +0300220
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300221 output_encoding => $encoding,
222 );
223
224$twig->parsefile($ARGV[0]);
225
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300226
227
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300228
229
230###########
231# END MAIN
232###########
233
234
235
236
237##############################
238# S U B R O U T I N E S
239##############################
240
241# sub set_dtd [
242# my $twig, $dtd = @_;
243# my $internal = qq|\nPUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"\n"DTD/xhtml1-strict.dtd"|;
244#
245# $twig->twig_doctype('html', undef, undef, $internal);
246# }
247
248
249
250sub root {
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300251 my ($twig, $root, $corpusHeader) =@_;
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300252
253 $root->set_gi('teiCorpus');
254 $root->set_att("xmlns", 'http://www.tei-c.org/ns/1.0');
255
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300256 &insertCorpusHeader($root, $corpusHeader);
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300257}
258
259
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300260
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300261sub insertCorpusHeader{
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300262 my ($root, $corpusHeader) =@_;
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300263
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300264 #---------------------------------------------------------------------------
265 # get some metadata for the current output corpus based on source and year
266 #---------------------------------------------------------------------------
267
Harald Lüngenccd84902024-08-27 16:03:47 +0300268 my @array = split(/\//, $ARGV[0]);
269 my $l = scalar(@array);
270 my $source = $array[$l-1];
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300271 $source =~ s/([0-9][0-9][0-9][0-9])\.xml$//;
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300272
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300273 my $language="Finnish";
274 my $lang_tla="fi";
275
Harald Lüngen86cbd932024-09-10 15:52:18 +0300276 my $year = $1; # $1 now containts substring in first bracket in regex above
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300277
Harald Lüngen86cbd932024-09-10 15:52:18 +0300278 my $ctitle = $source . " " . $year . ", from ". $kielipankkiCorpus . " for EuReCo"; # to do: also get name of corpus (klk-fi-v2-vrt)
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300279
280
281 #-----------------------
282 # set corpus header
283 #-----------------------
284
Harald Lüngen86cbd932024-09-10 15:52:18 +0300285 &set_title( $corpusHeader, $source, $year, $kielipankkiCorpus);
286 &set_sourceDesc($corpusHeader, $source, $year, $kielipankkiCorpus);
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300287
288 my $teiCorpusHeader = $corpusHeader->paste("first_child", $root);
289
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300290}
291
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300292
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300293#----------------------------
294# handler &text for <text>
295#----------------------------
296
297sub text {
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300298 my ($twig, $text, $textHeader) = @_;
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300299
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300300 $textcounter++;
Harald Lüngencaab0802024-08-23 17:28:22 +0300301
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300302
Harald Lüngencaab0802024-08-23 17:28:22 +0300303 # ToDo: catch all other, unexpected children of root
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300304
305 #--------------------------------------------------------------------------
306 # Get text metadata (attributes of <text>) and create teiHeader for <text>
307 #--------------------------------------------------------------------------
308
309 my $textattsref = $text->atts(); # $textattsref is now a reference to a hash and should be used with '->'
310
Harald Lüngen86cbd932024-09-10 15:52:18 +0300311 # &createTextHeader returns the $textID:
312 my $textID = &createTextHeader($text, $textattsref, $textHeader);
313
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300314 #--------------------------
315 # create <TEI> from <text>
316 #--------------------------
317
318 # set vrt <text> to <TEI> and delete all attributes after they were were saved above
319 $text->del_atts;
320 $text->set_gi("TEI");
Harald Lüngen86cbd932024-09-10 15:52:18 +0300321 $text->set_att('xml:id', $textID);
Harald Lüngen7abb0b52024-09-05 16:26:57 +0300322
323
Harald Lüngen86cbd932024-09-10 15:52:18 +0300324
325
Harald Lüngen7abb0b52024-09-05 16:26:57 +0300326
Harald Lüngencaab0802024-08-23 17:28:22 +0300327
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300328 #------------------------------------------------------------------
329 # create the <tei:text>, <body>, <div> elements inside <TEI>
330 #------------------------------------------------------------------
331
332 my $ttext_element = XML::Twig::Elt->new('text');
333 my $body_element = XML::Twig::Elt->new('body');
334 my $div_element = XML::Twig::Elt->new('div');
335
336 # set atts
Harald Lüngencaab0802024-08-23 17:28:22 +0300337 $div_element ->set_att("type", "page"); # ToDo: this is specific to KLK
338 $ttext_element->set_att("xml:lang", 'fi'); # as in ICC-NOR
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300339
340 # paste
341 $ttext_element->paste('last_child', $text);
342 $body_element ->paste('last_child', $ttext_element);
343 $div_element ->paste('last_child', $body_element);
344
345
346 #-------------------------------
347 # create <p> from <paragraph>
348 #-------------------------------
349
350 my @paragraphs = $text->children( 'paragraph');
351
352 foreach my $paragraph (@paragraphs) {
353
354 &setP($paragraph);
355
356 $paragraph->move('last_child', $div_element);
357
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300358 #------------------------------
359 # create <s> from <sentence>
360 #------------------------------
361
362 my @sentences = $paragraph->children('sentence');
363 foreach my $sentence (@sentences) {
364
365 &setS($sentence);
366
367
368 #--------------------------------------
369 # create <w> (word) from each $line
370 #--------------------------------------
371
372 my @lines = split(/\n+/, $sentence->xml_text);
373 $sentence->set_text("\n");
374
375 for my $line (@lines){ # Todo: Reihenfolge checken
376 if($line ne "" ){
377 my $w_element = XML::Twig::Elt->new('w');
378 &createW($w_element, $line);
379 $w_element->paste('last_child', $sentence);
380 }
Harald Lüngencaab0802024-08-23 17:28:22 +0300381 } # end words
382 } # end sentences
383 } # end paragraphs
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300384
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300385 # $twig->set_pretty_print( 'record');
Harald Lüngena20e69d2024-08-29 13:33:08 +0300386 # $twig->flush($OUT);
387 $twig->flush("/dev/stdout");
388}
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300389
390sub createTextHeader{
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300391 my ($text, $textattsref, $textHeader) = @_;
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300392
393 # USE 01 binding_id="2246025"
394 # USE 02 date="2021-01-15"
395 # 03 datefrom="20210115"
396 # 04 dateto="20210115"
397 # 05 elec_date="_"
398 # 06 file=""
399 # USE 07 filename_metadata="finclarin_siirto_k2021/0039-5552/2021/alto/2246025_0039-5552_2021-01-15_SK0221_mets.xml"
400 # USE 08 filename_orig ="finclarin_siirto_k2021/0039-5552/2021/alto/2246025_0039-5552_2021-01-15_SK0221_page-2021011502210030301.xml
401 # USE 09 id="t-bcd0f3fa-bbd3dac4"
402 # 10 img_url=""
403 # USE 11 issue_date="15.01.2021"
404 # USE 12 issue_no="SK0221"
405 # USE 13 issue_title="Suomen Kuvalehti"
406 # USE 14 label="Suomen Kuvalehti no. SK0221 15.01.2021"
407 # USE 16 language="fi"
408 # USE 17 page_id="p1"
409 # USE 18 page_no="None"
410 # 19 part_name="_"
411 # 20 publ_id="0039-5552"
412 # 21 publ_part=""
413 # USE 22 publ_title="Suomen Kuvalehti"
414 # USE 23 publ_type="aikakausi"
415 # USE 24 sentcount="70"
416 # USE 25 sum_lang="|xxx:44|fin:23|eng:3|"
417 # 26 timefrom="000000"
418 # 27 timeto="235959"
419 # USE 28 tokencount="304"
420 # 29 version_added="KLK-fi-2021">
421
422
423 my $BID = $textattsref->{'binding_id'};
424 my $DATE = $textattsref->{'date'};
425 my $METAFILENAME = $textattsref->{'filename_metadata'};
426 my $ORIGFILENAME = $textattsref->{'filename_orig'};
427 my $ID = $textattsref->{'id'};
428 my $ISSUEDATE = $textattsref->{'issue_date'};
429 my $ISSUENO = $textattsref->{'issue_no'};
430 my $ISSUETITLE = $textattsref->{'issue_title'};
431 my $LABEL = $textattsref->{'label'};
432 my $LANGUAGE = $textattsref->{'language'};
433 my $PAGEID = $textattsref->{'page_id'};
434 my $PAGENO = $textattsref->{'page_no'};
435 my $PUBLTITLE = $textattsref->{'publ_title'};
436 my $PUBLTYPE = $textattsref->{'publ_type'};
437 my $SENTCOUNT = $textattsref->{'sentcount'};
438 my $SUMLANG = $textattsref->{'sum_lang'};
439 my $TOKENCOUNT = $textattsref->{'tokencount'};
440
441
442 #-----------------------------
443 # Derived Metadata variables
444 #-----------------------------
445
446 my @datearray = split("-", $DATE);
447 my @langarray = split("|", $SUMLANG);
448 my @namearray = split(/[\.\/]/, $ORIGFILENAME); # use $namearray[4] as ID for the page
449
450
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300451
Harald Lüngen86cbd932024-09-10 15:52:18 +0300452 #----------------------------------------------------
453 # create textSigle to be returned from this function
454 #----------------------------------------------------
455
456 # SUK21.JAN.00001
457
458 my $corpusID = "SUK"; # ToDo read Table with Source metadata
459 my $yy = substr($datearray[0], 2, 2); # substr EXPR,OFFSET,LENGTH
460 my $mm = $datearray[1]; # substr EXPR,OFFSET,LENGTH
461 my $MMM = $months{$mm};
462
463 my $textID = $corpusID . $yy . "_" . $MMM . "." . sprintf("%05d", $doccounter{$mm}++);
464
465
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300466
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300467 #-----------------------------------------------------------------------
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300468 # CREATE text-teiHeader ACCORDING TO THE SKELETON in $textHeader
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300469 #-----------------------------------------------------------------------
470
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300471
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300472 $textHeader->paste('first_child', $text);
473
474 #-----------------------------------------------
475 # <teiHeader>
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300476 # <fileDesc n="EuReCo-KLK-FIN_[$ID]">
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300477 # <titleStmt>
478 # <title>[$LABEL, page $PAGENO]</title>
479
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300480 $textHeader->first_child("fileDesc") -> set_att('n', "EuReCo-". $kielipankkiCorpus . $ID);
481
482 $textHeader->first_child("fileDesc") -> first_child("titleStmt")->first_child("title")
483 ->set_text($LABEL . ", Text #" . $textcounter); # Case KLK; PAGENO scheint meist "None" zu sein
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300484
485 #-----------------------------------------------
486 # <fileDesc>
487 # <sourceDesc>
488 # <biblStruct>
489 # <analytic>
490 # <title type="main">[$LABEL, page $PAGENO]</title>
491 # <date>[$DATE]</date>
492 # <date type="year">TODO</date>
493 # <date type="month">TODO</date>
494 # <date type="day">TODO</date>
495 # <idno type="PAGEID">$PAGEID</idno>
496 # <idno type="BINDINGID">$BID</idno>
497 # <idno type="ID">$ID</idno>
498 # <idno type="KIELIPANKKI_METAFILENAME">$METAFILENAME</idno>
499 # <idno type="KIELIPANKKI_ORIGFILENAME">$ORIGFILENAME</idno>
500 # <textLang>$LANGUAGE</textLang>
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300501
502 my $analytic = $textHeader->get_xpath("./fileDesc/sourceDesc/biblStruct/analytic", 0);
503
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300504 $analytic->first_child("title") ->set_text($LABEL . ", Text #" . $textcounter); # Case KLK; PAGENO scheint meist "None" zu sein
505 $analytic->get_xpath('./date[@type="date"]', 0) ->set_text($DATE);
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300506 $analytic->get_xpath('./date[@type="year"]', 0) ->set_text($datearray[0]);
507 $analytic->get_xpath('./date[@type="month"]', 0) ->set_text($datearray[1]);
508 $analytic->get_xpath('./date[@type="day"]', 0) ->set_text($datearray[2]);
509 $analytic->get_xpath('./idno[@type="PAGEID"]', 0) ->set_text($PAGEID);
510 $analytic->get_xpath('./idno[@type="BINDINGID"]', 0) ->set_text($BID);
511 $analytic->get_xpath('./idno[@type="ID"]', 0) ->set_text($ID);
512 $analytic->get_xpath('./idno[@type="KIELIPANKKI_METAFILENAME"]', 0) ->set_text($METAFILENAME);
513 $analytic->get_xpath('./idno[@type="KIELIPANKKI_ORIGFILENAME"]', 0) ->set_text($ORIGFILENAME);
514 $analytic->first_child('textLang') ->set_text($LANGUAGE);
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300515
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300516 # <monogr>
517 # <title>$PUBLTITLE</title>
518 # <imprint>
519 # <pubPlace>TODO</pubPlace>
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300520 # <publisher>TODO</publisher>
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300521 # </imprint>
522 # <biblScope unit="ISSUETITLE"/>
523 # <biblScope unit="ISSUENO"/>
524 # <biblScope unit="ISSUEDATE"/>
525 # <biblScope unit="pp">$PAGENO</biblScope>
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300526
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300527 my $monogr = $textHeader->get_xpath("./fileDesc/sourceDesc/biblStruct/monogr", 0);
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300528
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300529 $monogr->first_child("title") ->set_text($PUBLTITLE);
530 $monogr->first_child("imprint")->first_child("pubPlace") ->set_text("ToDo"); # imprint is needed for tei validity
531 $monogr->first_child("imprint")->first_child("publisher") ->set_text("ToDo"); # imprint is needed for tei validity
532 $monogr->get_xpath('./biblScope[@unit="ISSUETITLE"]', 0) ->set_text($ISSUETITLE);
533 $monogr->get_xpath('./biblScope[@unit="ISSUENO"]', 0) ->set_text($ISSUENO);
534 $monogr->get_xpath('./biblScope[@unit="ISSUEDATE"]', 0) ->set_text($ISSUEDATE);
535 $monogr->get_xpath('./biblScope[@unit="pp"]', 0) ->set_text($PAGENO); # Achtung - PAGENO scheint meist "None" zu sein
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300536
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300537 # <encodingDesc>
538 # <tagsDecl>
539 # <namespace name="http://www.tei-c.org/ns/1.0">
540 # <tagUsage gi="s" occurs="SENTCOUNT"/>
541 # <tagUsage gi="w" occurs="TOKENCOUNT"/>
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300542
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300543 $textHeader->get_xpath('./encodingDesc/tagsDecl/namespace/tagUsage[@gi="s"]', 0) -> set_att('occurs', $SENTCOUNT);
544 $textHeader->get_xpath('./encodingDesc/tagsDecl/namespace/tagUsage[@gi="w"]', 0) -> set_att('occurs', $TOKENCOUNT);
545
546 # <profileDesc>
547 # <langUsage>
548 # <language ident="fi" usage="|xxx:44|fin:23|eng:3|"/>
549 # </langUsage>
550 # <textClass>
551 # <classCode scheme="kielipankki_klk">$PUBLTYPE</classCode>
552 # <classCode scheme="kielipankki_klk_mapped">TODO</classCode>
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300553
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300554 $textHeader->get_xpath('./profileDesc/langUsage/language', 0) ->set_att('ident', $LANGUAGE);
555 $textHeader->get_xpath('./profileDesc/langUsage/language', 0) ->set_att('usage', $SUMLANG);
556 # in @usage muss eigt. ein integer; am besten inhalt von SUMLANG aufdroeseln und mehrere <language> machen
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300557
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300558 $textHeader->get_xpath('./profileDesc/textClass/classCode[@scheme="kielipankki_klk"]', 0) ->set_text($PUBLTYPE);
559 $textHeader->get_xpath('./profileDesc/textClass/classCode[@scheme="kielipankki_klk_mapped"]', 0)->set_text("ToDo");
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300560
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300561 # <revisionDesc>
562 # <change when="TODO" who="HL">TEI version for EuReCo</change>
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300563
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300564 $textHeader->get_xpath('./revisionDesc/change', 0) ->set_att('when', localtime->ymd('-'));
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300565
Harald Lüngen86cbd932024-09-10 15:52:18 +0300566 return $textID;
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300567
Harald Lüngen86cbd932024-09-10 15:52:18 +0300568
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300569 #-----------------------------------
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300570 # END OF CREATING TEIHEADER
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300571 #-----------------------------------
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300572
573}
Harald Lüngen695ac1d2024-09-05 08:55:21 +0300574
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300575sub setP {
576 my ($paragraph) = @_;
577
578 $paragraph->set_gi('p');
579
580 # <paragraph id="p-bcd0f3fa-bbd3dac4-815ead7a" sum_lang="|fin:1|">
581 # atts of <paragraph>:
582 # @id USE
583 # @sum_lang USE: put in xml:lang and prefix the value with "x-" for private value
584
585 $paragraph->set_att("xml:lang", "x-" . $paragraph->att("sum_lang"));
586 $paragraph->del_att("sum_lang");
Harald Lüngena20e69d2024-08-29 13:33:08 +0300587 # $paragraph->change_att_name('id', 'xml:id');
588 $paragraph->del_att("id"); # diese id ist auch nicht eindeutig!!
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300589}
590sub setS {
591 my ($sentence) = @_;
592
593 $sentence->set_gi('s');
594
595 # the atts of <sentence>:
596 # USE 1 @id="s-bcd0f3fa-bbd3dac4-f7429090"
597 # USE 2 @lang="fin" -> xml:lang
598 # ? 3 @lang_conf="0.6734853"> -> ToDo @cert ?
599
600 # set attrs of <s>
601 $sentence->set_att("xml:lang", $sentence->att("lang")); # ToDo: convert the value / introduce a hash for lookup (input values: "fin", "xxx", ....)
Harald Lüngenfaf8d482024-08-27 21:19:47 +0300602 # $sentence->change_att_name('id', 'xml:id'); # nicht eindeutig
603 $sentence->del_att('id');
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300604 $sentence->del_att("lang"); # replaced by xml:lang
605 $sentence->del_att("lang_conf"); # for the time being
606
607}
608
609sub createW {
610 my ($w_element, $line) = @_;
611
612 #---------------------------
613 # Get the tags (=columns)
614 #---------------------------
615
616 my @tags = split(/\t/, $line);
617
618 # set content of <w> i.e. the token
619 $w_element->set_text($tags[0]);
620
621 # vrt positional-attributes in corpus KLK:
622 # USE [0] word
623 # USE [1] ref (id for reference of dephead)
624 # USE [2] lemma
625 # ? [3] lemmacomp (lemma with compound info - could go in @norm, as tag abuse?)
626 # USE [4] pos
627 # USE [5] msd
628 # USE [6] dephead
629 # USE [7] deprel
630 # [8] content (ocr-process)
631 # [9] vpos (ocr-process)
632 # [10] ocr (ocr-process)
633 # [11] cc (ocr-process)
634 # [12] hyph (ocr-process)
635 # [13] style (ocr-process)
636 # [14] lex (korp semantic disambiguation from G"oteborg)
637
638 # set the attributes of <w>:
639 $w_element->set_att("n", $tags[1]);
640 # $w_element->set_att("id", "w_" . $namearray[4] . $sentence->att("xml:id") . "_" . $tags[1]);
641 # so zusammengebaute ID ist auch nicht eindeutig...
Harald Lüngenfaf8d482024-08-27 21:19:47 +0300642 $w_element->del_att("id");
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300643 $w_element->set_att("lemma", $tags[2]);
644 # $w_element->set_att("norm", $tags[3]); # tag abuse of @norm
645 $w_element->set_att("pos", $tags[4]);
646 $w_element->set_att("msd", $tags[5]);
Harald Lüngenccd84902024-08-27 16:03:47 +0300647#TMP $w_element->set_att("head", $tags[6]);
648#TMP $w_element->set_att("deprel", $tags[7]);
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300649
650}
651
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300652
653sub set_title{
Harald Lüngen86cbd932024-09-10 15:52:18 +0300654 my ($corpusHeader, $source, $year, $kielipankkiCorpus) = @_;
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300655
Harald Lüngen86cbd932024-09-10 15:52:18 +0300656 my $cTitleString = $source . " " . $year . ", from ". $kielipankkiCorpus . " for EuReCo";
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300657
658 #<teiHeader>
659 # <fileDesc>
660 # <titleStmt>
661 # <title>[Aamulehti2021] from [klk-fi-v2-vrt for] EuReCo</title>
662 # </titleStmt>
663 # <!-- ... -->
664 # </fileDesc>
665 #</teiHeader>
666
667 my $cTitleNode = $corpusHeader->first_child("fileDesc")->first_child("titleStmt")->first_child("title");
668
669 $cTitleNode->set_text($cTitleString);
670
671}
672
673sub set_sourceDesc{
Harald Lüngen86cbd932024-09-10 15:52:18 +0300674 my ($corpusHeader, $source, $year, $kielipankkiCorpus) = @_;
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300675
Harald Lüngen86cbd932024-09-10 15:52:18 +0300676 my $cBiblString = $source . " " . $year . ", from ". $kielipankkiCorpus . " for EuReCo";
Harald Lüngendb5e6e72024-09-04 17:41:18 +0300677
678 #<teiHeader>
679 # <fileDesc>
680 # <!-- ... -->
681 # <sourceDesc>
682 # <bibl>[Aamulehti2021] from [klk-fi-v2-vrt]</bibl>
683 # </sourceDesc>
684 # <!-- ... -->
685 # </fileDesc>
686 #</teiHeader>
687
688 my $cBiblNode = $corpusHeader->first_child("fileDesc")->first_child("sourceDesc")->first_child("bibl");
689
690 $cBiblNode->set_text($cBiblString);
691}
692
693
694
695
696
697
698
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300699#################
700## usage_message
701#################
702
703
704sub usage_message {
Harald Lüngena7e91622024-08-23 17:33:11 +0300705 print " Usage: ./vrt2tei.pl <file.vrt.xml> <outfile>\n";
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300706 print " <file.vrt.xml> is a VRT file converted to proper XML\n";
707 exit;
708}
709
710