Add --base-tokens to write base/tokens.xml in conllu2korapxml
Change-Id: I66abe3117dd8e2f17aa883720cc936f4dd05de84
diff --git a/script/conllu2korapxml b/script/conllu2korapxml
index c187918..32cfbcf 100755
--- a/script/conllu2korapxml
+++ b/script/conllu2korapxml
@@ -28,7 +28,7 @@
'base-text' => \(my $base_text = 0),
'log|l=s' => \(my $log_level = 'warn'),
'output|o=s' => \(my $outh = '-'),
-
+ 'base-tokens' => \(my $base_tokens = 0),
'help|h' => sub {
pod2usage(
-verbose => 99,
@@ -85,8 +85,9 @@
my @dep_buffer;
my ($write_morpho, $write_syntax, $base) = (1, 0, 0);
-# Sentence texts accumulated per document for data.xml generation
my @doc_texts;
+my @token_spans;
+my $token_counter = 0;
my $filename;
my $first=1;
my @conllu_files = @ARGV;
@@ -125,6 +126,8 @@
$sentence_text = '';
$compute_offsets_mode = 0;
@doc_texts = ();
+ @token_spans = ();
+ $token_counter = 0;
} elsif (/^#\s*newdoc\s+id\s*=\s*(.*)/ && $text_sigle_template) {
my $newdoc_id = $1;
$newdoc_id =~ s/\s+$//;
@@ -311,6 +314,10 @@
</fs>
</span>
);
+ # Collect token span for optional base/tokens.xml output
+ if ($base_tokens) {
+ push @token_spans, [$spansFrom[$t], $spansTo[$t]];
+ }
$i++;
} else {
# Empty line = end of sentence
@@ -370,7 +377,6 @@
newZipStream($parser_file);
$zip->print($parse, qq(</spanList>\n</layer>\n));
}
- # Write data.xml with reconstructed text when --base-text is set
if ($base_text && $filename && @doc_texts) {
my $text_dir = dirname($filename);
$text_dir =~ s@(.*)/[^/]+$@$1@;
@@ -391,6 +397,18 @@
);
@doc_texts = ();
}
+ if ($base_tokens && $filename && @token_spans) {
+ my $text_dir = dirname($filename);
+ $text_dir =~ s@(.*)/[^/]+$@$1@;
+ my $tokens_path = "$text_dir/base/tokens.xml";
+ my $tokens_out = layer_header($docid);
+ for my $idx (0 .. $#token_spans) {
+ my ($from, $to) = @{$token_spans[$idx]};
+ $tokens_out .= qq( <span id="t_$idx" from="$from" to="$to"/>\n);
+ }
+ newZipStream($tokens_path);
+ $zip->print($tokens_out, qq(</spanList>\n</layer>\n));
+ }
}
sub layer_header {