Add --base-tokens to write base/tokens.xml in conllu2korapxml
Change-Id: I66abe3117dd8e2f17aa883720cc936f4dd05de84
diff --git a/Changes b/Changes
index 284f068..aa7fc08 100644
--- a/Changes
+++ b/Changes
@@ -7,6 +7,8 @@
- Add --base-text option to generate data.xml with
reconstructed plain text from 'text' comment lines
(implemented with AI assistance)
+ - Add --base-tokens option to generate base/tokens.xml
+ (implemented with AI assistance)
0.6.3 2024-06-04
- Trim filenames to fix double space after filename metadata
diff --git a/script/conllu2korapxml b/script/conllu2korapxml
index c187918..32cfbcf 100755
--- a/script/conllu2korapxml
+++ b/script/conllu2korapxml
@@ -28,7 +28,7 @@
'base-text' => \(my $base_text = 0),
'log|l=s' => \(my $log_level = 'warn'),
'output|o=s' => \(my $outh = '-'),
-
+ 'base-tokens' => \(my $base_tokens = 0),
'help|h' => sub {
pod2usage(
-verbose => 99,
@@ -85,8 +85,9 @@
my @dep_buffer;
my ($write_morpho, $write_syntax, $base) = (1, 0, 0);
-# Sentence texts accumulated per document for data.xml generation
my @doc_texts;
+my @token_spans;
+my $token_counter = 0;
my $filename;
my $first=1;
my @conllu_files = @ARGV;
@@ -125,6 +126,8 @@
$sentence_text = '';
$compute_offsets_mode = 0;
@doc_texts = ();
+ @token_spans = ();
+ $token_counter = 0;
} elsif (/^#\s*newdoc\s+id\s*=\s*(.*)/ && $text_sigle_template) {
my $newdoc_id = $1;
$newdoc_id =~ s/\s+$//;
@@ -311,6 +314,10 @@
</fs>
</span>
);
+ # Collect token span for optional base/tokens.xml output
+ if ($base_tokens) {
+ push @token_spans, [$spansFrom[$t], $spansTo[$t]];
+ }
$i++;
} else {
# Empty line = end of sentence
@@ -370,7 +377,6 @@
newZipStream($parser_file);
$zip->print($parse, qq(</spanList>\n</layer>\n));
}
- # Write data.xml with reconstructed text when --base-text is set
if ($base_text && $filename && @doc_texts) {
my $text_dir = dirname($filename);
$text_dir =~ s@(.*)/[^/]+$@$1@;
@@ -391,6 +397,18 @@
);
@doc_texts = ();
}
+ if ($base_tokens && $filename && @token_spans) {
+ my $text_dir = dirname($filename);
+ $text_dir =~ s@(.*)/[^/]+$@$1@;
+ my $tokens_path = "$text_dir/base/tokens.xml";
+ my $tokens_out = layer_header($docid);
+ for my $idx (0 .. $#token_spans) {
+ my ($from, $to) = @{$token_spans[$idx]};
+ $tokens_out .= qq( <span id="t_$idx" from="$from" to="$to"/>\n);
+ }
+ newZipStream($tokens_path);
+ $zip->print($tokens_out, qq(</spanList>\n</layer>\n));
+ }
}
sub layer_header {
diff --git a/t/ud_conllu.t b/t/ud_conllu.t
index 4e189c5..aba57a8 100644
--- a/t/ud_conllu.t
+++ b/t/ud_conllu.t
@@ -702,4 +702,111 @@
fail("data.xml single sentence: text matches exactly");
}
+# Test: --base-tokens produces base/tokens.xml
+my $zipcontent_tok = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud', '--base-tokens', $btx_file ],
+ { stdout => \$zipcontent_tok },
+ "conllu2korapxml runs with --base-tokens"
+);
+
+my $zipfile_tok = "$test_tempdir/test_tokens.zip";
+if ($zipcontent_tok) {
+ open(my $zfh, '>:raw', $zipfile_tok) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_tok;
+ close($zfh);
+
+ my $ziplist = `$UNZIP -l $zipfile_tok 2>/dev/null`;
+ like($ziplist,
+ qr@TEST/TEST/TEST_BTX_001/base/tokens\.xml@,
+ "Zip contains base/tokens.xml at correct path");
+
+ my $tokens_xml = `$UNZIP -p $zipfile_tok 'TEST/TEST/TEST_BTX_001/base/tokens.xml' 2>/dev/null`;
+
+ like($tokens_xml,
+ qr/docid="TEST_TEST\.TEST_BTX_001"/,
+ "tokens.xml has correct docid attribute");
+
+ like($tokens_xml,
+ qr/xmlns="http:\/\/ids-mannheim\.de\/ns\/KorAP"/,
+ "tokens.xml has correct namespace");
+
+ like($tokens_xml,
+ qr/<\?xml-model href="span\.rng"/,
+ "tokens.xml has correct processing instruction");
+
+ like($tokens_xml,
+ qr/version="KorAP-0\.4"/,
+ "tokens.xml has correct version");
+
+ # Token spans with sequential IDs and correct offsets
+ # Sentence 1: Geras(0..5) rytas(6..11) .(11..12)
+ like($tokens_xml,
+ qr/<span id="t_0" from="0" to="5"\/>/,
+ "Token t_0: Geras from=0 to=5");
+
+ like($tokens_xml,
+ qr/<span id="t_1" from="6" to="11"\/>/,
+ "Token t_1: rytas from=6 to=11");
+
+ like($tokens_xml,
+ qr/<span id="t_2" from="11" to="12"\/>/,
+ "Token t_2: period from=11 to=12");
+
+ # Sentence 2: Kaip(13..17) sekasi(18..24) ?(24..25)
+ like($tokens_xml,
+ qr/<span id="t_3" from="13" to="17"\/>/,
+ "Token t_3: Kaip from=13 to=17");
+
+ like($tokens_xml,
+ qr/<span id="t_4" from="18" to="24"\/>/,
+ "Token t_4: sekasi from=18 to=24");
+
+ like($tokens_xml,
+ qr/<span id="t_5" from="24" to="25"\/>/,
+ "Token t_5: question mark from=24 to=25");
+
+ # Self-closing span elements (no child fs element)
+ unlike($tokens_xml,
+ qr/<span id="t_0"[^\/]*>.*?<\/span>/s,
+ "Token spans are self-closing (no child elements)");
+}
+else {
+ fail("Zip contains base/tokens.xml at correct path");
+ fail("tokens.xml has correct docid attribute");
+ fail("tokens.xml has correct namespace");
+ fail("tokens.xml has correct processing instruction");
+ fail("tokens.xml has correct version");
+ fail("Token t_0: Geras from=0 to=5");
+ fail("Token t_1: rytas from=6 to=11");
+ fail("Token t_2: period from=11 to=12");
+ fail("Token t_3: Kaip from=13 to=17");
+ fail("Token t_4: sekasi from=18 to=24");
+ fail("Token t_5: question mark from=24 to=25");
+ fail("Token spans are self-closing (no child elements)");
+}
+
+# Test: without --base-tokens, no base/tokens.xml is produced
+my $zipcontent_notok = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud', $btx_file ],
+ { stdout => \$zipcontent_notok },
+ "conllu2korapxml runs without --base-tokens"
+);
+
+my $zipfile_notok = "$test_tempdir/test_notokens.zip";
+if ($zipcontent_notok) {
+ open(my $zfh, '>:raw', $zipfile_notok) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_notok;
+ close($zfh);
+
+ my $ziplist_notok = `$UNZIP -l $zipfile_notok 2>/dev/null`;
+ unlike($ziplist_notok,
+ qr/base\/tokens\.xml/,
+ "Zip does NOT contain base/tokens.xml when --base-tokens omitted");
+}
+else {
+ fail("Zip does NOT contain base/tokens.xml when --base-tokens omitted");
+}
+
done_testing;