Add automatic offset computation from 'text' comments
Change-Id: I25a6378ca4678ec317fc840d6f6a57b765701bd6
diff --git a/Changes b/Changes
index 9f2fdf9..f963676 100644
--- a/Changes
+++ b/Changes
@@ -1,3 +1,8 @@
+0.7.0 2025-06-03
+ - conllu2korapxml:
+ - Add automatic offset computation from 'text' comments
+ (implemented with AI assistance)
+
0.6.3 2024-06-04
- Trim filenames to fix double space after filename metadata
- Set permissions of zip contents to 666
diff --git a/script/conllu2korapxml b/script/conllu2korapxml
index c8e4ec0..a68aced 100755
--- a/script/conllu2korapxml
+++ b/script/conllu2korapxml
@@ -14,7 +14,7 @@
my %opts;
my %processedFilenames;
-our $VERSION = '0.6.3';
+our $VERSION = '0.7.0';
our $VERSION_MSG = "\nconllu2korapxml - v$VERSION\n";
use constant {
@@ -59,6 +59,12 @@
my @spansTo;
my $current;
my ($unknown, $known) = (0, 0);
+my $sentence_text = '';
+my $doc_offset = 0;
+my $text_pos = 0;
+my $compute_offsets_mode = 0;
+# Buffer dependency arcs until all token offsets in a sentence are known
+my @dep_buffer;
my ($write_morpho, $write_syntax, $base) = (1, 0, 0);
my $filename;
@@ -95,6 +101,9 @@
}
$processedFilenames{$filename}=1;
$i=0;
+ $doc_offset = 0;
+ $sentence_text = '';
+ $compute_offsets_mode = 0;
} elsif(/^#\s*foundry\s*[:=]\s*(.*)/) {
if(!$foundry_name) {
$dependency_foundry_name = $foundry_name = $1;
@@ -134,8 +143,43 @@
@spansFrom = split(/\s+/, $1);
} elsif (/^(?:#|0\.4)\s+(?:end_offsets|to)\s+[:=]\s*(.*)/) {
@spansTo = split(/\s+/, $1);
+ } elsif (/^#\s*text\s*=\s*(.*)/) {
+ # Store sentence text for automatic offset computation
+ $sentence_text = decode('UTF-8', $1);
}
} elsif ( !/^\s*$/ ) {
+ # Pre-split columns before offset computation needs the raw form
+ my @raw_cols = split('\t');
+ chomp $raw_cols[$#raw_cols] if @raw_cols;
+
+ # Enter offset computation mode when no explicit offsets given
+ if (!$compute_offsets_mode && scalar @spansFrom == 0
+ && $sentence_text && $docid) {
+ $compute_offsets_mode = 1;
+ @spansFrom = ();
+ @spansTo = ();
+ # Sentence-level span covers the full sentence text
+ $spansFrom[0] = $doc_offset;
+ $spansTo[0] = $doc_offset + length($sentence_text);
+ $text_pos = 0;
+ }
+
+ # Locate each token form in sentence text via index()
+ if ($compute_offsets_mode && @raw_cols >= 2 && $raw_cols[0] =~ /^\d+$/) {
+ my $raw_form = decode('UTF-8', $raw_cols[1]);
+ my $t_num = $raw_cols[0];
+ # Find token starting from current position (handles SpaceAfter)
+ my $pos_in_text = index($sentence_text, $raw_form, $text_pos);
+ if ($pos_in_text >= 0) {
+ $spansFrom[$t_num] = $doc_offset + $pos_in_text;
+ $spansTo[$t_num] = $spansFrom[$t_num] + length($raw_form);
+ # Advance past this token for the next search
+ $text_pos = $pos_in_text + length($raw_form);
+ } else {
+ $log->warn("WARNING: Token form not found in sentence text in $conllu_file line $.");
+ }
+ }
+
if ( !$docid || scalar @spansTo == 0 || scalar @spansFrom == 0 ) {
if ( !$docid ) {
$log->warn("WARNING: Invalid input in $conllu_file: text_id (e.g. '# text_id = GOE_AGA.00000') missing in line $. when writing to $outh");
@@ -155,8 +199,7 @@
$s =~ s/</</g;
$s =~ s/>/>/g;
$s;
- } split('\t');
- chomp $parsed[9];
+ } @raw_cols;
if (@parsed != 10) {
$log->warn("WARNING: skipping strange parser output line in $docid");
$i++;
@@ -169,14 +212,8 @@
}
if($parsed[6] =~ /\d+/ && $parsed[7] !~ /_/) {
$write_syntax=1;
- my $from=$spansFrom[$parsed[6]];
- my $to=$spansTo[$parsed[6]];
- $parse .= qq@<span id="s${s}_n$t" from="$spansFrom[$t]" to="$spansTo[$t]">
-<rel label="$parsed[7]">
-<span from="$from" to="$to"/>
-</rel>
-</span>
-@;
+ # Buffer dep arcs; flushed at end of sentence when offsets are ready
+ push @dep_buffer, [$s, $t, $parsed[6], $parsed[7]];
}
my $pos = $parsed[4];
my $upos = $parsed[3];
@@ -208,6 +245,17 @@
</span>
);
$i++;
+ } else {
+ # Empty line = end of sentence
+ flush_dep_buffer();
+ if ($compute_offsets_mode) {
+ # Advance doc offset past sentence + 1-char space separator
+ $doc_offset += length($sentence_text) + 1;
+ @spansFrom = ();
+ @spansTo = ();
+ $compute_offsets_mode = 0;
+ }
+ $sentence_text = '';
}
}
$current .= "\n";
@@ -230,7 +278,22 @@
}
}
+# Write buffered dependency arcs now that all token offsets are resolved
+sub flush_dep_buffer {
+ foreach my $dep (@dep_buffer) {
+ my ($ds, $dt, $dhead, $dlabel) = @$dep;
+ $parse .= qq@<span id="s${ds}_n$dt" from="$spansFrom[$dt]" to="$spansTo[$dt]">
+<rel label="$dlabel">
+<span from="$spansFrom[$dhead]" to="$spansTo[$dhead]"/>
+</rel>
+</span>
+@;
+ }
+ @dep_buffer = ();
+}
+
sub closeDoc {
+ flush_dep_buffer();
if ($write_morpho && $morpho_file) {
newZipStream($morpho_file);
$zip->print($morpho, qq( </spanList>\n</layer>\n));
diff --git a/t/ud_conllu.t b/t/ud_conllu.t
new file mode 100644
index 0000000..bec7f29
--- /dev/null
+++ b/t/ud_conllu.t
@@ -0,0 +1,370 @@
+use strict;
+use warnings;
+use Test::More;
+use Test::Script;
+use Test::TempDir::Tiny;
+
+my $UNZIP = `sh -c 'command -v unzip'`;
+chomp $UNZIP;
+
+if ($UNZIP eq '') {
+ plan skip_all => 'No unzip executable found in PATH.';
+}
+
+my $offset_data = <<'CONLLU';
+# filename = TEST/TEST/TEST_OFF_001/base/tokens.xml
+# text_id = TEST_TEST.TEST_OFF_001
+# text = Geras rytas.
+1 Geras geras ADJ bdv. Case=Nom 2 amod _ _
+2 rytas rytas NOUN dkt. Case=Nom 0 root _ _
+3 . . PUNCT skyr. _ 2 punct _ SpaceAfter=No
+
+# text = Kaip sekasi?
+1 Kaip kaip ADV prv. _ 2 advmod _ _
+2 sekasi sektis VERB vksm. _ 0 root _ SpaceAfter=No
+3 ? ? PUNCT skyr. _ 2 punct _ _
+
+CONLLU
+
+my $test_tempdir = tempdir();
+my $offset_file = "$test_tempdir/test_offset.conllu";
+{
+ open(my $ofh, '>:encoding(UTF-8)', $offset_file)
+ or die "Cannot write test file: $!";
+ print $ofh $offset_data;
+ close($ofh);
+}
+
+my $zipcontent_off = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud', $offset_file ],
+ { stdout => \$zipcontent_off },
+ "conllu2korapxml computes offsets from # text"
+);
+
+my $zipfile_off = "$test_tempdir/test_offset.zip";
+if ($zipcontent_off) {
+ open(my $zfh, '>:raw', $zipfile_off) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_off;
+ close($zfh);
+
+ my $morpho_xml = `$UNZIP -p $zipfile_off 'TEST/TEST/TEST_OFF_001/ud/morpho.xml' 2>/dev/null`;
+
+ # Sentence 1: "Geras rytas." (12 chars, starts at 0)
+ # Geras: 0-5, rytas: 6-11, .: 11-12
+ like($morpho_xml,
+ qr/id="s1_n1" from="0" to="5"/,
+ "Computed offset: token 'Geras' at 0..5");
+ like($morpho_xml,
+ qr/id="s1_n2" from="6" to="11"/,
+ "Computed offset: token 'rytas' at 6..11");
+ like($morpho_xml,
+ qr/id="s1_n3" from="11" to="12"/,
+ "Computed offset: token '.' at 11..12 (SpaceAfter=No)");
+
+ # Sentence 2: "Kaip sekasi?" (12 chars, starts at 13 = 12 + 1 space)
+ # Kaip: 13-17, sekasi: 18-24, ?: 24-25
+ like($morpho_xml,
+ qr/id="s2_n1" from="13" to="17"/,
+ "Computed offset: token 'Kaip' at 13..17 (sentence 2)");
+ like($morpho_xml,
+ qr/id="s2_n2" from="18" to="24"/,
+ "Computed offset: token 'sekasi' at 18..24");
+ like($morpho_xml,
+ qr/id="s2_n3" from="24" to="25"/,
+ "Computed offset: token '?' at 24..25 (SpaceAfter=No)");
+
+ # Verify dependency XML has correct offsets (head references)
+ my $dep_xml = `$UNZIP -p $zipfile_off 'TEST/TEST/TEST_OFF_001/ud/dependency.xml' 2>/dev/null`;
+
+ # Token 1 "Geras" (0..5) -> head token 2 "rytas" (6..11)
+ like($dep_xml,
+ qr/id="s1_n1" from="0" to="5".*?<span from="6" to="11"/s,
+ "Dependency: 'Geras' head points to 'rytas' offsets");
+
+ # Token 2 "rytas" (6..11) -> head 0 = sentence span (0..12)
+ like($dep_xml,
+ qr/id="s1_n2" from="6" to="11".*?<span from="0" to="12"/s,
+ "Dependency: 'rytas' head points to sentence span");
+}
+else {
+ fail("Computed offset: token 'Geras' at 0..5");
+ fail("Computed offset: token 'rytas' at 6..11");
+ fail("Computed offset: token '.' at 11..12 (SpaceAfter=No)");
+ fail("Computed offset: token 'Kaip' at 13..17 (sentence 2)");
+ fail("Computed offset: token 'sekasi' at 18..24");
+ fail("Computed offset: token '?' at 24..25 (SpaceAfter=No)");
+ fail("Dependency: 'Geras' head points to 'rytas' offsets");
+ fail("Dependency: 'rytas' head points to sentence span");
+}
+
+# No SpaceAfter at all - every token has normal space separation
+my $no_spaceafter_data = <<'CONLLU';
+# filename = TEST/TEST/TEST_NSA_001/base/tokens.xml
+# text_id = TEST_TEST.TEST_NSA_001
+# text = One two three
+1 One one NUM num. _ 0 root _ _
+2 two two NUM num. _ 1 flat _ _
+3 three three NUM num. _ 1 flat _ _
+
+CONLLU
+
+my $nsa_file = "$test_tempdir/test_no_spaceafter.conllu";
+{
+ open(my $nfh, '>:encoding(UTF-8)', $nsa_file)
+ or die "Cannot write test file: $!";
+ print $nfh $no_spaceafter_data;
+ close($nfh);
+}
+
+my $zipcontent_nsa = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud', $nsa_file ],
+ { stdout => \$zipcontent_nsa },
+ "conllu2korapxml handles tokens with no SpaceAfter"
+);
+
+my $zipfile_nsa = "$test_tempdir/test_no_spaceafter.zip";
+if ($zipcontent_nsa) {
+ open(my $zfh, '>:raw', $zipfile_nsa) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_nsa;
+ close($zfh);
+
+ my $morpho_nsa = `$UNZIP -p $zipfile_nsa 'TEST/TEST/TEST_NSA_001/ud/morpho.xml' 2>/dev/null`;
+
+ # "One two three" = 13 chars
+ # One: 0-3, two: 4-7, three: 8-13
+ like($morpho_nsa,
+ qr/id="s1_n1" from="0" to="3"/,
+ "No SpaceAfter: token 'One' at 0..3");
+ like($morpho_nsa,
+ qr/id="s1_n2" from="4" to="7"/,
+ "No SpaceAfter: token 'two' at 4..7");
+ like($morpho_nsa,
+ qr/id="s1_n3" from="8" to="13"/,
+ "No SpaceAfter: token 'three' at 8..13");
+}
+else {
+ fail("No SpaceAfter: token 'One' at 0..3");
+ fail("No SpaceAfter: token 'two' at 4..7");
+ fail("No SpaceAfter: token 'three' at 8..13");
+}
+
+# Adjacent tokens - no space between consecutive tokens (SpaceAfter=No)
+my $adjacent_data = <<'CONLLU';
+# filename = TEST/TEST/TEST_ADJ_001/base/tokens.xml
+# text_id = TEST_TEST.TEST_ADJ_001
+# text = foo(bar)baz end
+1 foo foo NOUN n. _ 0 root _ SpaceAfter=No
+2 ( ( PUNCT skyr. _ 3 punct _ SpaceAfter=No
+3 bar bar NOUN n. _ 1 appos _ SpaceAfter=No
+4 ) ) PUNCT skyr. _ 3 punct _ SpaceAfter=No
+5 baz baz NOUN n. _ 1 conj _ _
+6 end end NOUN n. _ 1 conj _ _
+
+CONLLU
+
+my $adj_file = "$test_tempdir/test_adjacent.conllu";
+{
+ open(my $afh, '>:encoding(UTF-8)', $adj_file)
+ or die "Cannot write test file: $!";
+ print $afh $adjacent_data;
+ close($afh);
+}
+
+my $zipcontent_adj = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud', $adj_file ],
+ { stdout => \$zipcontent_adj },
+ "conllu2korapxml handles adjacent tokens without spaces"
+);
+
+my $zipfile_adj = "$test_tempdir/test_adjacent.zip";
+if ($zipcontent_adj) {
+ open(my $zfh, '>:raw', $zipfile_adj) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_adj;
+ close($zfh);
+
+ my $morpho_adj = `$UNZIP -p $zipfile_adj 'TEST/TEST/TEST_ADJ_001/ud/morpho.xml' 2>/dev/null`;
+
+ # "foo(bar)baz end" = 15 chars
+ # foo: 0-3, (: 3-4, bar: 4-7, ): 7-8, baz: 8-11, end: 12-15
+ like($morpho_adj,
+ qr/id="s1_n1" from="0" to="3"/,
+ "Adjacent: token 'foo' at 0..3");
+ like($morpho_adj,
+ qr/id="s1_n2" from="3" to="4"/,
+ "Adjacent: token '(' at 3..4 (no space before)");
+ like($morpho_adj,
+ qr/id="s1_n3" from="4" to="7"/,
+ "Adjacent: token 'bar' at 4..7 (no space before)");
+ like($morpho_adj,
+ qr/id="s1_n4" from="7" to="8"/,
+ "Adjacent: token ')' at 7..8 (no space before)");
+ like($morpho_adj,
+ qr/id="s1_n5" from="8" to="11"/,
+ "Adjacent: token 'baz' at 8..11 (no space before)");
+ like($morpho_adj,
+ qr/id="s1_n6" from="12" to="15"/,
+ "Adjacent: token 'end' at 12..15 (space before)");
+}
+else {
+ fail("Adjacent: token 'foo' at 0..3");
+ fail("Adjacent: token '(' at 3..4 (no space before)");
+ fail("Adjacent: token 'bar' at 4..7 (no space before)");
+ fail("Adjacent: token ')' at 7..8 (no space before)");
+ fail("Adjacent: token 'baz' at 8..11 (no space before)");
+ fail("Adjacent: token 'end' at 12..15 (space before)");
+}
+
+# Repeated token form - same word appears multiple times in sentence
+my $repeat_data = <<'CONLLU';
+# filename = TEST/TEST/TEST_REP_001/base/tokens.xml
+# text_id = TEST_TEST.TEST_REP_001
+# text = the cat and the dog
+1 the the DET det. _ 2 det _ _
+2 cat cat NOUN n. _ 0 root _ _
+3 and and CCONJ cc. _ 5 cc _ _
+4 the the DET det. _ 5 det _ _
+5 dog dog NOUN n. _ 2 conj _ _
+
+CONLLU
+
+my $rep_file = "$test_tempdir/test_repeat.conllu";
+{
+ open(my $rfh, '>:encoding(UTF-8)', $rep_file)
+ or die "Cannot write test file: $!";
+ print $rfh $repeat_data;
+ close($rfh);
+}
+
+my $zipcontent_rep = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud', $rep_file ],
+ { stdout => \$zipcontent_rep },
+ "conllu2korapxml handles repeated token forms"
+);
+
+my $zipfile_rep = "$test_tempdir/test_repeat.zip";
+if ($zipcontent_rep) {
+ open(my $zfh, '>:raw', $zipfile_rep) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_rep;
+ close($zfh);
+
+ my $morpho_rep = `$UNZIP -p $zipfile_rep 'TEST/TEST/TEST_REP_001/ud/morpho.xml' 2>/dev/null`;
+
+ # "the cat and the dog" = 19 chars
+ # the: 0-3, cat: 4-7, and: 8-11, the: 12-15, dog: 16-19
+ like($morpho_rep,
+ qr/id="s1_n1" from="0" to="3"/,
+ "Repeated: first 'the' at 0..3");
+ like($morpho_rep,
+ qr/id="s1_n2" from="4" to="7"/,
+ "Repeated: 'cat' at 4..7");
+ like($morpho_rep,
+ qr/id="s1_n4" from="12" to="15"/,
+ "Repeated: second 'the' at 12..15 (not matching first)");
+ like($morpho_rep,
+ qr/id="s1_n5" from="16" to="19"/,
+ "Repeated: 'dog' at 16..19");
+}
+else {
+ fail("Repeated: first 'the' at 0..3");
+ fail("Repeated: 'cat' at 4..7");
+ fail("Repeated: second 'the' at 12..15 (not matching first)");
+ fail("Repeated: 'dog' at 16..19");
+}
+
+# Single-token sentence - minimal sentence with only one word
+my $single_data = <<'CONLLU';
+# filename = TEST/TEST/TEST_SNG_001/base/tokens.xml
+# text_id = TEST_TEST.TEST_SNG_001
+# text = Hello
+1 Hello hello INTJ intj. _ 0 root _ _
+
+# text = World
+1 World world NOUN n. _ 0 root _ _
+
+CONLLU
+
+my $sng_file = "$test_tempdir/test_single.conllu";
+{
+ open(my $sfh, '>:encoding(UTF-8)', $sng_file)
+ or die "Cannot write test file: $!";
+ print $sfh $single_data;
+ close($sfh);
+}
+
+my $zipcontent_sng = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud', $sng_file ],
+ { stdout => \$zipcontent_sng },
+ "conllu2korapxml handles single-token sentences"
+);
+
+my $zipfile_sng = "$test_tempdir/test_single.zip";
+if ($zipcontent_sng) {
+ open(my $zfh, '>:raw', $zipfile_sng) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_sng;
+ close($zfh);
+
+ my $morpho_sng = `$UNZIP -p $zipfile_sng 'TEST/TEST/TEST_SNG_001/ud/morpho.xml' 2>/dev/null`;
+
+ # Sentence 1: "Hello" (5 chars, starts at 0)
+ # Sentence 2: "World" (5 chars, starts at 6 = 5 + 1 space)
+ like($morpho_sng,
+ qr/id="s1_n1" from="0" to="5"/,
+ "Single-token: 'Hello' at 0..5");
+ like($morpho_sng,
+ qr/id="s2_n1" from="6" to="11"/,
+ "Single-token: 'World' at 6..11 (after space separator)");
+}
+else {
+ fail("Single-token: 'Hello' at 0..5");
+ fail("Single-token: 'World' at 6..11 (after space separator)");
+}
+
+# Explicit offsets take precedence over # text when both are present
+my $explicit_wins_data = <<'CONLLU';
+# filename = TEST/TEST/TEST_EXP_001/base/tokens.xml
+# text_id = TEST_TEST.TEST_EXP_001
+# text = Geras rytas.
+# start_offsets = 0 100 200 300
+# end_offsets = 999 199 299 399
+1 Geras geras ADJ bdv. Case=Nom 2 amod _ _
+2 rytas rytas NOUN dkt. Case=Nom 0 root _ _
+3 . . PUNCT skyr. _ 2 punct _ _
+
+CONLLU
+
+my $exp_file = "$test_tempdir/test_explicit.conllu";
+{
+ open(my $efh, '>:encoding(UTF-8)', $exp_file)
+ or die "Cannot write test file: $!";
+ print $efh $explicit_wins_data;
+ close($efh);
+}
+
+my $zipcontent_exp = '';
+script_runs(
+ [ 'script/conllu2korapxml', '-f', 'ud', $exp_file ],
+ { stdout => \$zipcontent_exp },
+ "conllu2korapxml uses explicit offsets when both # text and offsets present"
+);
+
+my $zipfile_exp = "$test_tempdir/test_explicit.zip";
+if ($zipcontent_exp) {
+ open(my $zfh, '>:raw', $zipfile_exp) or die "Cannot write zip: $!";
+ print $zfh $zipcontent_exp;
+ close($zfh);
+
+ my $morpho_exp = `$UNZIP -p $zipfile_exp 'TEST/TEST/TEST_EXP_001/ud/morpho.xml' 2>/dev/null`;
+ like($morpho_exp,
+ qr/id="s1_n1" from="100" to="199"/,
+ "Explicit offsets win: token uses from=100 (not computed 0)");
+}
+else {
+ fail("Explicit offsets win: token uses from=100 (not computed 0)");
+}
+
+done_testing;