Add --text-sigle option and 'newdoc' id support for standard UD CoNLL-U input
Change-Id: I76277c47ca35a9b1263ab7433ba3e4d41d44124f
diff --git a/Changes b/Changes
index f963676..b9f012d 100644
--- a/Changes
+++ b/Changes
@@ -1,7 +1,9 @@
-0.7.0 2025-06-03
+0.7.0 2026-06-08
- conllu2korapxml:
- Add automatic offset computation from 'text' comments
(implemented with AI assistance)
+ - new option --text-sigle <template> to derive text sigle from newdoc id
+ (implemented with AI assistance)
0.6.3 2024-06-04
- Trim filenames to fix double space after filename metadata
diff --git a/script/conllu2korapxml b/script/conllu2korapxml
index a68aced..b1ebfb5 100755
--- a/script/conllu2korapxml
+++ b/script/conllu2korapxml
@@ -24,6 +24,7 @@
GetOptions(
'force-foundry|f=s' => \(my $foundry_name = ''),
+ 'text-sigle|s=s' => \(my $text_sigle_template = ''),
'log|l=s' => \(my $log_level = 'warn'),
'output|o=s' => \(my $outh = '-'),
@@ -44,6 +45,22 @@
}
);
+# Validate text-sigle template format
+if ($text_sigle_template) {
+ my $test_sigle = $text_sigle_template;
+ $test_sigle =~ s/\{ID\}/PLACEHOLDER/gi;
+ my @tpl_parts = split('/', $test_sigle);
+ if (scalar @tpl_parts != 3) {
+ die "ERROR: text-sigle template must produce exactly 3"
+ . " slash-separated parts (CORPUS/DOC/TEXT), got "
+ . scalar(@tpl_parts) . " from '$text_sigle_template'\n";
+ }
+ if (grep { $_ eq '' } @tpl_parts) {
+ die "ERROR: text-sigle template '$text_sigle_template'"
+ . " must not contain empty parts\n";
+ }
+}
+
# Establish logger
binmode(STDERR, ':encoding(UTF-8)');
Log::Any::Adapter->set('Stderr', log_level => $log_level);
@@ -104,6 +121,48 @@
$doc_offset = 0;
$sentence_text = '';
$compute_offsets_mode = 0;
+ } elsif (/^#\s*newdoc\s+id\s*=\s*(.*)/ && $text_sigle_template) {
+ my $newdoc_id = $1;
+ $newdoc_id =~ s/\s+$//;
+
+ if (!$first) {
+ closeDoc(0);
+ } else {
+ $first = 0;
+ }
+
+ my $text_sigle = $text_sigle_template;
+ $text_sigle =~ s/\{ID\}/$newdoc_id/gi;
+
+ my @parts = split('/', $text_sigle);
+ if (scalar @parts != 3 || grep { $_ eq '' } @parts) {
+ die "ERROR: Expanded text sigle '$text_sigle' is not"
+ . " in valid CORPUS/DOC/TEXT format\n";
+ }
+
+ $filename = "$text_sigle/base/tokens.xml";
+ if ($processedFilenames{$filename}) {
+ $log->warn("WARNING: $filename is already processed");
+ }
+ $processedFilenames{$filename} = 1;
+ $i = 0;
+
+ my $last = pop @parts;
+ $docid = join('_', @parts) . '.' . $last;
+
+ my $docSigle = $docid;
+ $docSigle =~ s/\..*//;
+ if ($docSigle ne $lastDocSigle) {
+ $log->info("Analyzing $docSigle");
+ $lastDocSigle = $docSigle;
+ }
+ $known = $unknown = 0;
+ $current = "";
+
+ $morpho_file = "$text_sigle/$foundry_name/morpho.xml";
+ $parser_file = "$text_sigle/$dependency_foundry_name/dependency.xml";
+
+ $parse = $morpho = layer_header($docid);
} elsif(/^#\s*foundry\s*[:=]\s*(.*)/) {
if(!$foundry_name) {
$dependency_foundry_name = $foundry_name = $1;
@@ -344,6 +403,12 @@
Set foundry name and ignore foundry names in the input.
+=item B<--text-sigle|-s>
+
+Text sigle template with C<{ID}> placeholder for standard UD CoNLL-U
+input. When C<# newdoc id = E<lt>IDE<gt>> is encountered, C<{ID}> is
+replaced by the document id to derive the KorAP-XML text sigle,
+filename, and docid.
=item B<--help|-h>
diff --git a/t/ud_conllu.t b/t/ud_conllu.t
index bec7f29..4e0a0a7 100644
--- a/t/ud_conllu.t
+++ b/t/ud_conllu.t
@@ -367,4 +367,149 @@
fail("Explicit offsets win: token uses from=100 (not computed 0)");
}
+# -------------------------------------------------------------------
+# Inline test data: minimal UD CoNLL-U with explicit offsets.
+# Uses "# newdoc id" (UD style) instead of "# filename" / "# text_id"
+# (KorAP style). Explicit offsets are provided so this test does not
+# depend on automatic offset computation.
+# -------------------------------------------------------------------
+
+my $ud_conllu_data = <<'CONLLU';
+# newdoc id = TEST_LIT_001
+# start_offsets = 0 0 6 11
+# end_offsets = 12 5 11 12
+1 Geras geras ADJ bdv. Case=Nom 2 amod _ _
+2 rytas rytas NOUN dkt. Case=Nom 0 root _ _
+3 . . PUNCT skyr. _ 2 punct _ _
+
+CONLLU
+
+my $conllu_file = "$test_tempdir/test_ud.conllu";
+{
+ open(my $fh, '>:encoding(UTF-8)', $conllu_file)
+ or die "Cannot write test file: $!";
+ print $fh $ud_conllu_data;
+ close($fh);
+}
+
+my $zipcontent = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud',
+ '--text-sigle', 'TEST/TEST/{ID}', $conllu_file ],
+ { stdout => \$zipcontent },
+ "conllu2korapxml accepts --text-sigle with UD CoNLL-U input"
+);
+
+my $zipfile = "$test_tempdir/test_newdoc.zip";
+if ($zipcontent) {
+ open(my $zfh, '>:raw', $zipfile) or die "Cannot write zip: $!";
+ print $zfh $zipcontent;
+ close($zfh);
+
+ my $ziplist = `$UNZIP -l $zipfile 2>/dev/null`;
+ like($ziplist,
+ qr@TEST/TEST/TEST_LIT_001/ud/morpho\.xml@,
+ "Zip contains morpho.xml at path derived from newdoc id");
+ like($ziplist,
+ qr@TEST/TEST/TEST_LIT_001/ud/dependency\.xml@,
+ "Zip contains dependency.xml at path derived from newdoc id");
+
+ my $zipdata = `$UNZIP -c $zipfile 2>/dev/null`;
+ like($zipdata,
+ qr/docid="TEST_TEST\.TEST_LIT_001"/,
+ "docid correctly derived from text-sigle template and newdoc id");
+}
+else {
+ fail("Zip contains morpho.xml at path derived from newdoc id");
+ fail("Zip contains dependency.xml at path derived from newdoc id");
+ fail("docid correctly derived from text-sigle template and newdoc id");
+}
+
+my $zipcontent_lc = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud',
+ '--text-sigle', 'TEST/TEST/{id}', $conllu_file ],
+ { stdout => \$zipcontent_lc },
+ "text-sigle template accepts {id} (lowercase)"
+);
+
+if ($zipcontent_lc) {
+ my $zipfile_lc = "$test_tempdir/test_lc.zip";
+ open(my $zfh, '>:raw', $zipfile_lc) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_lc;
+ close($zfh);
+
+ my $zipdata_lc = `$UNZIP -c $zipfile_lc 2>/dev/null`;
+ like($zipdata_lc,
+ qr/docid="TEST_TEST\.TEST_LIT_001"/,
+ "docid correct with lowercase {id} template");
+}
+else {
+ fail("docid correct with lowercase {id} template");
+}
+
+my $zipcontent_mc = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud',
+ '--text-sigle', 'TEST/TEST/{Id}', $conllu_file ],
+ { stdout => \$zipcontent_mc },
+ "text-sigle template accepts {Id} (mixed case)"
+);
+
+if ($zipcontent_mc) {
+ my $zipfile_mc = "$test_tempdir/test_mc.zip";
+ open(my $zfh, '>:raw', $zipfile_mc) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_mc;
+ close($zfh);
+
+ my $zipdata_mc = `$UNZIP -c $zipfile_mc 2>/dev/null`;
+ like($zipdata_mc,
+ qr/docid="TEST_TEST\.TEST_LIT_001"/,
+ "docid correct with mixed-case {Id} template");
+}
+else {
+ fail("docid correct with mixed-case {Id} template");
+}
+
+# Template with only 2 parts (missing corpus level)
+my $err_2parts = `$^X script/conllu2korapxml -f ud --text-sigle 'TEST/{ID}' $conllu_file 2>&1`;
+isnt($? >> 8, 0, "Rejects template with only 2 parts (non-zero exit)");
+like($err_2parts, qr/ERROR/, "Error message for 2-part template");
+
+# Template with 4 parts (too many levels)
+my $err_4parts = `$^X script/conllu2korapxml -f ud --text-sigle 'A/B/C/{ID}' $conllu_file 2>&1`;
+isnt($? >> 8, 0, "Rejects template with 4 parts (non-zero exit)");
+like($err_4parts, qr/ERROR/, "Error message for 4-part template");
+
+# Template with empty middle part
+my $err_empty = `$^X script/conllu2korapxml -f ud --text-sigle 'TEST//{ID}' $conllu_file 2>&1`;
+isnt($? >> 8, 0, "Rejects template with empty part (non-zero exit)");
+like($err_empty, qr/ERROR/, "Error message for empty-part template");
+
+# Template with only 1 part (no slashes)
+my $err_1part = `$^X script/conllu2korapxml -f ud --text-sigle '{ID}' $conllu_file 2>&1`;
+isnt($? >> 8, 0, "Rejects template with only 1 part (non-zero exit)");
+like($err_1part, qr/ERROR/, "Error message for 1-part template");
+
+my $bad_id_data = <<'CONLLU';
+# newdoc id = BAD/SLASH_ID
+# start_offsets = 0 0
+# end_offsets = 4 4
+1 Test test NOUN NN _ 0 root _ _
+
+CONLLU
+
+my $bad_id_file = "$test_tempdir/bad_id.conllu";
+{
+ open(my $bfh, '>:encoding(UTF-8)', $bad_id_file)
+ or die "Cannot write test file: $!";
+ print $bfh $bad_id_data;
+ close($bfh);
+}
+
+my $err_slash = `$^X script/conllu2korapxml -f ud --text-sigle 'A/B/{ID}' $bad_id_file 2>&1`;
+isnt($? >> 8, 0,
+ "Rejects newdoc id with slash (expanded sigle has wrong part count)");
+like($err_slash, qr/ERROR/,
+ "Error message for newdoc id containing slash");
done_testing;