Gitiles
Code Review Sign In
korap.ids-mannheim.de / EuReCo / kielipankki4eureco / 9d4e04628f7112f9cc3a4939d8f192965d6f3981 / . / vrt2tei.pl
blob: c7789a8f9049d3c913f1c095cbe60371946654c9 [file] [log] [blame]
Harald Lüngen9d4e0462024-08-23 09:34:22 +0300[diff] [blame^]1#! /usr/bin/perl -w
2
3
4###########################################################################################################################################################
5# vrt2tei.pl
6# eureco
7# leibniz-institut fuer deutsche sprache / csc finland esbo
8# august 2024
9#
10#
11# using XML::Twig , see http://www.xmltwig.org/, https://metacpan.org/pod/XML::Twiga
12#
13# usage: see below the usage function
14# Usage: ./xml2i5.pl <vrtxmlfile.xml> <outfile>
15# <vrtxmlfile>: xml-ised vrt file
16#
17#
18# TODO:
19# 1 insert dtd spec, or ref to TEI
20
21# 3a UPLOAD in GITHUB
22# 3b add @head and @deprel to I5 sowie auch @msd
23# 3c bearbeitung von @head und @deprel in tei2korapxml durch Nils?
24# 3d build 30 billion corpus
25
26# 4a take care of IDs
27# 4b see to the values of @xml:lang
28# 5 abfangen von unerwarteten elementen dh andere als <sentence> und <paragraph>
29# 5a wort reihenfolge nochmal checken
30# 6 checks and balances
31# 6a output nach stdout machen
32# 7 How to encode Kielipankki and National Library of Finland? in teiCorpus Header
33# 8 construct <idsDoc>s for the months (or go for TEI)
34# 9 parallelisation in bash and application on sub corpora of KLK
35# 10 re-implementation of the gawk code in the perl script
36# 12 re-implement creation of text header from xml file in another twig / parametrize TEI vs I5
37
38
39
40#remember
41#formatted.xml:105613: element w: validity error : No declaration for attribute deprel of element w
42#formatted.xml:105613: element w: validity error : No declaration for attribute head of element w
43
44
45#
46#
47############################################################################################################################################################
48
49
50use strict;
51use warnings;
52
53use XML::Twig;
54use XML::Generator ':pretty'; # apparently no effect when using flush();
55
56
57use locale; # diese drei Zeilen, damit \b im regex nicht Umlaute und ß matcht.
58use POSIX qw(locale_h); # to be able to use setlocale()
59#setlocale(LC_ALL,'de_DE');
60setlocale(LC_ALL, "fi_FI");
61use utf8;
62use open qw( :std :encoding(UTF-8) );
63
64use Time::Piece;
65use Tie::IxHash;
66
67#----------------------
68# check file arguments:
69#----------------------
70
71# arg0 infile: vrt-xml
72# arg1 outfile: tei
73
74unless($ARGV[1]) {&usage_message()}; # min arg0 und arg1
75if ($ARGV[2]) {&usage_message()}; # max arg1
76
77
78
79
80####################
81# GLOBAL VARIABLES
82####################
83
84my $encoding = "UTF-8";
85#my $encoding = "iso-8859-1"; # dieses $encoding ist NUR fuer das output s.u. twig funktion
86
87
88
89#####################
90# M A I N
91#####################
92
93
94# open result file and initialise filehandle
95open(my $OUT, ">> $ARGV[1]") || die("cannot open file: $ARGV[1]");
96
97
98
99#-----------------------------------------------------------------------------------
100# start twig and call start tag handler for root and twig handler for each <text>
101#-----------------------------------------------------------------------------------
102
103my $twig="";
104
105$twig = new XML::Twig(
106 keep_spaces => 1, # dadurch auch whitespaces an ehemeligen elementgrenzen im output
107 keep_atts_order => 1, # requires Tie::IxHash
108 pretty_print => 'indented',
109 start_tag_handlers => {
110 texts => \&root
111 },
112 twig_handlers => {
113 text => \&text
114 },
115 # dtd_handlers => { # ToDo for I5
116 # \&set_dtd;
117 # }
118 output_encoding => $encoding,
119 );
120
121$twig->parsefile($ARGV[0]);
122
123close($OUT);
124
125
126###########
127# END MAIN
128###########
129
130
131
132
133##############################
134# S U B R O U T I N E S
135##############################
136
137# sub set_dtd [
138# my $twig, $dtd = @_;
139# my $internal = qq|\nPUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"\n"DTD/xhtml1-strict.dtd"|;
140#
141# $twig->twig_doctype('html', undef, undef, $internal);
142# }
143
144
145
146sub root {
147 my ($twig, $root) =@_;
148
149 $root->set_gi('teiCorpus');
150 $root->set_att("xmlns", 'http://www.tei-c.org/ns/1.0');
151
152 &insertCorpusHeader($root);
153}
154
155
156sub insertCorpusHeader{
157 my ($root) =@_;
158
159 my $teiHeader = $root ->insert_new_elt("first_child", 'teiHeader');
160 my $fileDesc = $teiHeader ->insert_new_elt("last_child", 'fileDesc');
161
162 my $titleStmt = $fileDesc ->insert_new_elt("last_child", 'titleStmt');
163 my $title = $titleStmt ->insert_new_elt("last_child", 'title');
164 $title ->set_text(" KLK-fi-2021 for EuReCo");
165
166 my $publicationStmt = $fileDesc ->insert_new_elt("last_child", 'publicationStmt');
167 my $distributor = $publicationStmt->insert_new_elt("last_child", 'distributor');
168 $distributor ->set_text("NOT FOR DISTRIBUTION - to be used locally in EuReCo");
169
170 my $sourceDesc = $fileDesc ->insert_new_elt("last_child", 'sourceDesc');
171 my $bibl = $sourceDesc ->insert_new_elt("last_child", 'bibl');
172 $bibl ->set_text("ToDo");
173
174}
175
176
177#----------------------------
178# handler &text for <text>
179#----------------------------
180
181sub text {
182
183 my ($twig, $text) = @_;
184
185 # ToDo: catch all other, unexpected children
186
187 #--------------------------------------------------------------------------
188 # Get text metadata (attributes of <text>) and create teiHeader for <text>
189 #--------------------------------------------------------------------------
190
191 my $textattsref = $text->atts(); # $textattsref is now a reference to a hash and should be used with '->'
192
193 &createTextHeader($text, $textattsref);
194
195 #--------------------------
196 # create <TEI> from <text>
197 #--------------------------
198
199 # set vrt <text> to <TEI> and delete all attributes after they were were saved above
200 $text->del_atts;
201 $text->set_gi("TEI");
202
203 #------------------------------------------------------------------
204 # create the <tei:text>, <body>, <div> elements inside <TEI>
205 #------------------------------------------------------------------
206
207 my $ttext_element = XML::Twig::Elt->new('text');
208 my $body_element = XML::Twig::Elt->new('body');
209 my $div_element = XML::Twig::Elt->new('div');
210
211 # set atts
212 $div_element->set_att("type", "page"); # ToDo: this is specific to KLK
213
214 # paste
215 $ttext_element->paste('last_child', $text);
216 $body_element ->paste('last_child', $ttext_element);
217 $div_element ->paste('last_child', $body_element);
218
219
220 #-------------------------------
221 # create <p> from <paragraph>
222 #-------------------------------
223
224 my @paragraphs = $text->children( 'paragraph');
225
226 foreach my $paragraph (@paragraphs) {
227
228 &setP($paragraph);
229
230 $paragraph->move('last_child', $div_element);
231
232
233 #------------------------------
234 # create <s> from <sentence>
235 #------------------------------
236
237 my @sentences = $paragraph->children('sentence');
238 foreach my $sentence (@sentences) {
239
240 &setS($sentence);
241
242
243 #--------------------------------------
244 # create <w> (word) from each $line
245 #--------------------------------------
246
247 my @lines = split(/\n+/, $sentence->xml_text);
248 $sentence->set_text("\n");
249
250 for my $line (@lines){ # Todo: Reihenfolge checken
251 if($line ne "" ){
252 my $w_element = XML::Twig::Elt->new('w');
253 &createW($w_element, $line);
254 $w_element->paste('last_child', $sentence);
255 }
256 }
257 }
258 }
259
260 $twig->flush($OUT);
261}
262
263
264sub createTextHeader{
265 my ($text, $textattsref) = @_;
266
267 # USE 01 binding_id="2246025"
268 # USE 02 date="2021-01-15"
269 # 03 datefrom="20210115"
270 # 04 dateto="20210115"
271 # 05 elec_date="_"
272 # 06 file=""
273 # USE 07 filename_metadata="finclarin_siirto_k2021/0039-5552/2021/alto/2246025_0039-5552_2021-01-15_SK0221_mets.xml"
274 # USE 08 filename_orig ="finclarin_siirto_k2021/0039-5552/2021/alto/2246025_0039-5552_2021-01-15_SK0221_page-2021011502210030301.xml
275 # USE 09 id="t-bcd0f3fa-bbd3dac4"
276 # 10 img_url=""
277 # USE 11 issue_date="15.01.2021"
278 # USE 12 issue_no="SK0221"
279 # USE 13 issue_title="Suomen Kuvalehti"
280 # USE 14 label="Suomen Kuvalehti no. SK0221 15.01.2021"
281 # USE 16 language="fi"
282 # USE 17 page_id="p1"
283 # USE 18 page_no="None"
284 # 19 part_name="_"
285 # 20 publ_id="0039-5552"
286 # 21 publ_part=""
287 # USE 22 publ_title="Suomen Kuvalehti"
288 # USE 23 publ_type="aikakausi"
289 # USE 24 sentcount="70"
290 # USE 25 sum_lang="|xxx:44|fin:23|eng:3|"
291 # 26 timefrom="000000"
292 # 27 timeto="235959"
293 # USE 28 tokencount="304"
294 # 29 version_added="KLK-fi-2021">
295
296
297 my $BID = $textattsref->{'binding_id'};
298 my $DATE = $textattsref->{'date'};
299 my $METAFILENAME = $textattsref->{'filename_metadata'};
300 my $ORIGFILENAME = $textattsref->{'filename_orig'};
301 my $ID = $textattsref->{'id'};
302 my $ISSUEDATE = $textattsref->{'issue_date'};
303 my $ISSUENO = $textattsref->{'issue_no'};
304 my $ISSUETITLE = $textattsref->{'issue_title'};
305 my $LABEL = $textattsref->{'label'};
306 my $LANGUAGE = $textattsref->{'language'};
307 my $PAGEID = $textattsref->{'page_id'};
308 my $PAGENO = $textattsref->{'page_no'};
309 my $PUBLTITLE = $textattsref->{'publ_title'};
310 my $PUBLTYPE = $textattsref->{'publ_type'};
311 my $SENTCOUNT = $textattsref->{'sentcount'};
312 my $SUMLANG = $textattsref->{'sum_lang'};
313 my $TOKENCOUNT = $textattsref->{'tokencount'};
314
315
316 #-----------------------------
317 # Derived Metadata variables
318 #-----------------------------
319
320 my @datearray = split("-", $DATE);
321 my @langarray = split("|", $SUMLANG);
322 my @namearray = split(/[\.\/]/, $ORIGFILENAME); # use $namearray[4] as ID for the page
323
324
325 #-----------------------------------------------------------------------
326 # CREATE text-teiHeader ACCORDING TO THE SKELETON in klk-header.tei.xml
327 #-----------------------------------------------------------------------
328
329 # create <teiHeader> inside <TEI>
330 my $teiHeader = XML::Twig::Elt->new('teiHeader');
331 $teiHeader->paste('first_child', $text);
332
333 ## insert_new_elt is a combo of new and paste, cf. xml::twig docu:
334 ## insert_new_elt ($opt_position, $gi, $opt_atts_hashref, @opt_content)
335
336 my $fileDesc = $teiHeader->insert_new_elt('fileDesc' => {n => "EuReCo_KLK-fi_" . $namearray[4]});
337 my $encodingDesc = $teiHeader->insert_new_elt("last_child", 'encodingDesc');
338 my $profileDesc = $teiHeader->insert_new_elt("last_child", 'profileDesc');
339 my $revisionDesc = $teiHeader->insert_new_elt("last_child", 'revisionDesc');
340
341 #---------------------
342 # fileDesc/titleStmt
343 #---------------------
344 my $titleStmt = $fileDesc ->insert_new_elt('titleStmt');
345 my $title = $titleStmt->insert_new_elt("last_child", 'title');
346 my $respStmt = $titleStmt->insert_new_elt("last_child", 'respStmt');
347 my $resp = $respStmt ->insert_new_elt("last_child", 'resp');
348 my $name = $respStmt ->insert_new_elt("last_child", 'name');
349
350 # set texts for titleStmt
351 $title->set_text($LABEL . ", page " . $PAGENO); # Achtung - PAGENO scheint meist "None" zu sein
352 $resp ->set_text("compiled by EuReCo");
353 $name ->set_text("EuReCo: HL");
354
355 #--------------------------
356 # fileDesc/publicationStmt
357 #--------------------------
358 my $publicationStmt = $fileDesc ->insert_new_elt("last_child", 'publicationStmt');
359 my $distributor = $publicationStmt->insert_new_elt("last_child", 'distributor');
360 my $note = $distributor ->insert_new_elt("last_child", 'note');
361 my $availability = $publicationStmt->insert_new_elt("last_child", 'availability');
362 my $licence = $availability ->insert_new_elt("last_child", 'licence');
363
364 # set texts for publicationStmt
365 $note ->set_text("NOT FOR DISTRIBUTION - to be used locally in EuReCo");
366 $licence->set_text("CLARIN-RES"); # TODO: Ausfuherlichere Licence info in KLK Metadata Record
367
368 #------------------------------
369 # fileDesc/sourceDesc/biblStruct
370 #------------------------------
371 my $sourceDesc = $fileDesc ->insert_new_elt("last_child", 'sourceDesc');
372 my $biblStruct = $sourceDesc->insert_new_elt("last_child", 'biblStruct');
373
374 # fileDesc/sourceDesc/biblStruct/analytic
375 my $analytic = $biblStruct->insert_new_elt("last_child", 'analytic');
376 my $analytic_title = $analytic->insert_new_elt("last_child", 'title' => {type => "main"} );
377# my $analytic_date = $analytic->insert_new_elt("last_child", 'date');
378 my $analytic_date_year = $analytic->insert_new_elt("last_child", 'date' => {type => "year"});
379 my $analytic_date_month = $analytic->insert_new_elt("last_child", 'date' => {type => "month"});
380 my $analytic_date_day = $analytic->insert_new_elt("last_child", 'date' => {type => "day"});
381 my $analytic_idno_pageid = $analytic->insert_new_elt("last_child", 'idno' => {type => "PAGEID"});
382 my $analytic_idno_bindingid = $analytic->insert_new_elt("last_child", 'idno' => {type => "BINDINGID"});
383 my $analytic_idno_id = $analytic->insert_new_elt("last_child", 'idno' => {type => "ID"});
384 my $analytic_idno_metafile = $analytic->insert_new_elt("last_child", 'idno' => {type => "KIELIPANKKI_METAFILENAME"});
385 my $analytic_idno_origfile = $analytic->insert_new_elt("last_child", 'idno' => {type => "KIELIPANKKI_ORIGFILENAME"});
386 my $analytic_textlang = $analytic->insert_new_elt("last_child", 'textLang');
387
388 # set texts for analytic
389 $analytic_title ->set_text($LABEL . ", page " . $PAGENO); # Achtung $PAGENO scheint meist "None zu sein"
390# $analytic_date ->set_text($DATE);
391 $analytic_date_year ->set_text($datearray[0]);
392 $analytic_date_month ->set_text($datearray[1]);
393 $analytic_date_day ->set_text($datearray[2]);
394 $analytic_idno_pageid ->set_text($PAGEID);
395 $analytic_idno_bindingid->set_text($BID);
396 $analytic_idno_id ->set_text($ID);
397 $analytic_idno_metafile ->set_text($METAFILENAME);
398 $analytic_idno_origfile ->set_text($ORIGFILENAME);
399 $analytic_textlang ->set_text($LANGUAGE);
400
401 #-------------------------------------
402 # fileDesc/sourceDesc/biblStruct/monogr
403 #-------------------------------------
404 my $monogr = $biblStruct->insert_new_elt("last_child", 'monogr');
405 my $monogr_title = $monogr ->insert_new_elt("last_child", 'title');
406 my $imprint = $monogr ->insert_new_elt("last_child", 'imprint'); # imprint is needed for valididty
407 my $pubPlace = $imprint ->insert_new_elt("last_child", 'pubPlace'); # imprint is needed for validity
408 my $publisher = $imprint ->insert_new_elt("last_child", 'publisher'); # imprint is needed for validity
409 my $biblScope_issuetitle = $monogr ->insert_new_elt("last_child", 'biblScope' => {unit => 'ISSUETITLE'} );
410 my $biblScope_issueno = $monogr ->insert_new_elt("last_child", 'biblScope' => {unit => 'ISSUENO'} );
411 my $biblScope_issuedate = $monogr ->insert_new_elt("last_child", 'biblScope' => {unit => 'ISSUEDATE'} );
412 my $biblScope_pp = $monogr ->insert_new_elt("last_child", 'biblScope' => {unit => 'PAGENO'} ); # Achtung PAGENO ist meist "None" ?
413
414 # set texts for monogr
415 $monogr_title ->set_text($PUBLTITLE);
416 $pubPlace ->set_text("TODO");
417 $publisher ->set_text("TODO");
418 $biblScope_issuetitle->set_text($ISSUETITLE);
419 $biblScope_issueno ->set_text($ISSUENO);
420 $biblScope_issuedate ->set_text($ISSUEDATE);
421 $biblScope_pp ->set_text($PAGENO);
422
423 #---------------
424 # encodingDesc
425 #---------------
426 my $tagsDecl = $encodingDesc->insert_new_elt("last_child", 'tagsDecl');
427 my $namespace = $tagsDecl ->insert_new_elt("last_child", 'namespace' => {name => 'http://www.tei-c.org/ns/1.0'});
428 my $tagUsage_s = $namespace ->insert_new_elt("last_child", 'tagUsage' => {gi => 's', occurs => $SENTCOUNT});
429 my $tagUsage_w = $namespace ->insert_new_elt("last_child", 'tagUsage' => {gi => 'w', occurs => $TOKENCOUNT});
430
431 #-------------
432 # profileDesc
433 #-------------
434 my $langUsage = $profileDesc ->insert_new_elt("last_child", 'langUsage');
435 my $language = $langUsage ->insert_new_elt("last_child", 'language' => {ident => $LANGUAGE, usage => $SUMLANG});
436 # Achtung in @usage muss eigt. ein integer; am besten inhalt von SUMLANG aufdroeseln und mehrere <language> machen
437 my $textClass = $profileDesc ->insert_new_elt("last_child", 'textClass');
438 my $classCode_fi = $textClass ->insert_new_elt("last_child", 'classCode' => {scheme => "KLK_PUBLTYPE"});
439# my $classCode_en = $textClass ->insert_new_elt("last_child", 'classCode' => {scheme => "KLK_PUBLTYPE_MAPPED"});
440
441 #---------------------------
442 # set texts for profileDesc
443 #---------------------------
444 $classCode_fi ->set_text($PUBLTYPE);
445# $classCode_en->set_text($PUBLTYPETRANSL);
446
447 #---------------
448 # revisionDesc
449 #---------------
450 my $change = $revisionDesc ->insert_new_elt("last_child", 'change' => {when => localtime->ymd('-'), who => 'HL' });
451
452 # set texts for revisionDesc
453 $change->set_text("TEI version for EuReCo");
454
455
456 ###################################
457 # END OF CREATING TEIHEADER
458 ###################################
459
460}
461
462sub setP {
463 my ($paragraph) = @_;
464
465 $paragraph->set_gi('p');
466
467 # <paragraph id="p-bcd0f3fa-bbd3dac4-815ead7a" sum_lang="|fin:1|">
468 # atts of <paragraph>:
469 # @id USE
470 # @sum_lang USE: put in xml:lang and prefix the value with "x-" for private value
471
472 $paragraph->set_att("xml:lang", "x-" . $paragraph->att("sum_lang"));
473 $paragraph->del_att("sum_lang");
474 $paragraph->change_att_name('id', 'xml:id');
475}
476sub setS {
477 my ($sentence) = @_;
478
479 $sentence->set_gi('s');
480
481 # the atts of <sentence>:
482 # USE 1 @id="s-bcd0f3fa-bbd3dac4-f7429090"
483 # USE 2 @lang="fin" -> xml:lang
484 # ? 3 @lang_conf="0.6734853"> -> ToDo @cert ?
485
486 # set attrs of <s>
487 $sentence->set_att("xml:lang", $sentence->att("lang")); # ToDo: convert the value / introduce a hash for lookup (input values: "fin", "xxx", ....)
488 $sentence->change_att_name('id', 'xml:id');
489 $sentence->del_att("lang"); # replaced by xml:lang
490 $sentence->del_att("lang_conf"); # for the time being
491
492}
493
494sub createW {
495 my ($w_element, $line) = @_;
496
497 #---------------------------
498 # Get the tags (=columns)
499 #---------------------------
500
501 my @tags = split(/\t/, $line);
502
503 # set content of <w> i.e. the token
504 $w_element->set_text($tags[0]);
505
506 # vrt positional-attributes in corpus KLK:
507 # USE [0] word
508 # USE [1] ref (id for reference of dephead)
509 # USE [2] lemma
510 # ? [3] lemmacomp (lemma with compound info - could go in @norm, as tag abuse?)
511 # USE [4] pos
512 # USE [5] msd
513 # USE [6] dephead
514 # USE [7] deprel
515 # [8] content (ocr-process)
516 # [9] vpos (ocr-process)
517 # [10] ocr (ocr-process)
518 # [11] cc (ocr-process)
519 # [12] hyph (ocr-process)
520 # [13] style (ocr-process)
521 # [14] lex (korp semantic disambiguation from G"oteborg)
522
523 # set the attributes of <w>:
524 $w_element->set_att("n", $tags[1]);
525 # $w_element->set_att("id", "w_" . $namearray[4] . $sentence->att("xml:id") . "_" . $tags[1]);
526 # so zusammengebaute ID ist auch nicht eindeutig...
527 $w_element->change_att_name('id', 'xml:id');
528 $w_element->set_att("lemma", $tags[2]);
529 # $w_element->set_att("norm", $tags[3]); # tag abuse of @norm
530 $w_element->set_att("pos", $tags[4]);
531 $w_element->set_att("msd", $tags[5]);
532 $w_element->set_att("head", $tags[6]);
533 $w_element->set_att("deprel", $tags[7]);
534
535}
536
537#################
538## usage_message
539#################
540
541
542sub usage_message {
543 print " Usage: ./xml2i5.pl <file.vrt.xml> <outfile>\n";
544 print " <file.vrt.xml> is a VRT file converted to proper XML\n";
545 exit;
546}
547
548