Add --word2vec option to produce one sentence per line output
otionally with leading metadata columns (via --extract-metadata-regex
<regex>)
Change-Id: Ic564c14bc08a0041798048a90e466b09ea24666e
diff --git a/script/korapxml2conllu b/script/korapxml2conllu
index bd27235..8fd898b 100755
--- a/script/korapxml2conllu
+++ b/script/korapxml2conllu
@@ -17,6 +17,14 @@
my %opts;
my %plain_texts;
my %sentence_ends;
+my %metadata;
+
+my $offsets = 1;
+my $comments = 1;
+my $extract_metadata = 0;
+my @extract_metadata_regex;
+
+my $lm_training_data = 0;
our $VERSION = '0.4.1.9000';
@@ -33,7 +41,11 @@
's-bounds-from-morpho' => \(my $s_bounds_from_morpho = 0),
'log|l=s' => \(my $log_level = 'warn'),
'columns|c=n' => \(my $columns = 10),
-
+ 'word2vec|lm-training-data!' => \$lm_training_data,
+ 'token-separator|s=s' => \(my $token_separator = "\n"),
+ 'offsets!' => \$offsets,
+ 'comments!' => \$comments,
+ 'extract-metadata-regex|m=s@' => \@extract_metadata_regex,
'help|h' => sub {
pod2usage(
-verbose => 99,
@@ -51,11 +63,23 @@
}
);
+if (@extract_metadata_regex) {
+ $extract_metadata = 1;
+ @extract_metadata_regex = split(/,/,join(',',@extract_metadata_regex));
+}
+
# Establish logger
binmode(STDERR, ':encoding(UTF-8)');
Log::Any::Adapter->set('Stderr', log_level => $log_level);
$log->notice('Debugging is activated') if DEBUG;
+if ($lm_training_data) {
+ $columns = 1;
+ $comments = 0;
+ $offsets = 0;
+ $token_separator = " ";
+}
+
my $docid="";
my ($current_id, $current_from, $current_to, $token);
my $current;
@@ -98,10 +122,14 @@
$baseOnly = $morpho_zip eq $data_zip;
my ($morphoOrTokenCommand, $plaintextAndStructureCommand);
+ my $zip_content_pattern = "[sd][ta]*";
+ if ($extract_metadata) {
+ $zip_content_pattern = "[sdh][tae]*";
+ }
if (!$baseOnly) {
$morphoOrTokenCommand = "$UNZIP -c $morpho_zip '*/${sigle_pattern}*/*/*/morpho.xml' $zipsiglepattern |";
if ($extract_attributes_regex || !$s_bounds_from_morpho) {
- $plaintextAndStructureCommand = "$UNZIP -c $data_zip '*/${sigle_pattern}*/*/[sd][ta]*.xml' $zipsiglepattern |";
+ $plaintextAndStructureCommand = "$UNZIP -c $data_zip '*/${sigle_pattern}*/*/$zip_content_pattern.xml' $zipsiglepattern |";
} else {
$log->debug("Not reading structure information.");
$plaintextAndStructureCommand = "$UNZIP -c $data_zip '*/${sigle_pattern}*/*/data.xml' $zipsiglepattern |";
@@ -116,12 +144,12 @@
}
$morphoOrTokenCommand =~ s/-l/-c/;
$morphoOrTokenCommand .= ' |';
- $plaintextAndStructureCommand = "$UNZIP -c $data_zip '*/${sigle_pattern}*/*/[sd][ta]*.xml' $zipsiglepattern |";
+ $plaintextAndStructureCommand = "$UNZIP -c $data_zip " . "'*/${sigle_pattern}*/*/$zip_content_pattern.xml' " . "$zipsiglepattern |";
}
open (MORPHO_OR_TOKENPIPE, $morphoOrTokenCommand) or die "cannot unzip $morpho_zip";
open (PLAINTEXTPIPE, $plaintextAndStructureCommand) or die "cannot unzip $data_zip";
- print "$COMMENT_START foundry = $foundry$COMMENT_END\n";
+ print "$COMMENT_START foundry = $foundry$COMMENT_END\n" if ($comments);
while (<MORPHO_OR_TOKENPIPE>) {
if (/^ inflating: (.*)/) {
$filename=$1;
@@ -150,7 +178,7 @@
last if(m@</layer>@);
}
}
- print STDOUT "$COMMENT_START filename = $filename$COMMENT_END\n$COMMENT_START text_id = $docid$COMMENT_END\n";
+ print STDOUT "$COMMENT_START filename = $filename$COMMENT_END\n$COMMENT_START text_id = $docid$COMMENT_END\n" if($comments);
$log->debug("Analyzing $docid");
} elsif (m@^\s*<f\s+.*name="([^"]+)">([^<]+)</f>@) {
if ($1 eq "lemma") {
@@ -199,7 +227,7 @@
$known++;
$conll[$ID_idx] = $#current_lines+1;
if ($columns == 1) {
- $current .= "$conll[1]\n";
+ $current .= "$conll[1]$token_separator" ;
} else {
$current .= join("\t", @conll[0..$columns-1]) . "\n"; # conll columns
}
@@ -208,6 +236,12 @@
$log->debug("Using sentence end for $docid \@$current_to");
$current .= "\n";
printTokenRanges();
+ if ($extract_metadata) {
+ for (my $i = 0; $i < @extract_metadata_regex; $i++) {
+ print "$metadata{$docid}[$i]\t";
+ }
+ }
+ $current =~ s/ $//;
print STDOUT $current;
$current = "";
$known = 0;
@@ -249,11 +283,12 @@
exit;
sub printTokenRanges {
+ return if(!$offsets);
print "$COMMENT_START start_offsets = ", $current_lines[0]->[0];
foreach my $t (@current_lines) {
print STDOUT " $t->[0]";
}
- print "$COMMENT_END\n$COMMENT_START end_offsets = ", $current_lines[$#current_lines]->[1];
+ print "$COMMENT_END\n$COMMENT_START end_offsets = ", $current_lines[$#current_lines]->[1] if($comments);
foreach my $t (@current_lines) {
print STDOUT " $t->[1]";
}
@@ -281,11 +316,23 @@
my $text_count = 0;
my ($current_id, $current_from, $current_to);
- if($plain_texts{$target_id} && ($s_bounds_from_morpho || $sentence_ends{$target_id})) {
+ if($plain_texts{$target_id} && ($s_bounds_from_morpho || $sentence_ends{$target_id}) && (!$extract_metadata || $metadata{$target_id})) {
$log->debug("Already got $target_id");
return 1;
}
while(<PLAINTEXTPIPE>) {
+ if ($extract_metadata) {
+ if (/<textSigle>([^<]+)/) {
+ $docid = $1;
+ $docid =~ s@/@_@;
+ $log->debug("textsigle=$docid");
+ }
+ for (my $i=0; $i < @extract_metadata_regex; $i++) {
+ if ($_ =~ /$extract_metadata_regex[$i]/) {
+ $metadata{$docid}[$i]=$1;
+ }
+ }
+ }
if(/<raw_text[^>]+docid="([^"]*)/) {
$docid=$1;
$log->debug("Getting plain text for $docid");
@@ -324,7 +371,7 @@
s/&/&/go;
tr/…•⋅»«ˮ“”„›‹ʼ‘’‚′‐‑‒–—―⁓⁻₋−﹣－/...""""""'''''''-/;
$plain_texts{$docid} = $_;
- last if(!$extract_attributes_regex && ($text_count++ > 1 && $plain_texts{$target_id}));
+ last if(!$extract_attributes_regex && ($text_count++ > 1 && $plain_texts{$target_id} && (!$extract_metadata || $metadata{$target_id})));
} elsif (m@<text>(.*)@) {
$_= decode("utf-8", $1, Encode::FB_DEFAULT);
s/</</go;
@@ -341,7 +388,7 @@
tr/…•⋅»«ˮ“”„›‹ʼ‘’‚′‐‑‒–—―⁓⁻₋−﹣－/...""""""'''''''-/;
$plain_texts{$docid} .= $_;
$text_started=0;
- last if(!$extract_attributes_regex && ($text_count++ > 1 && $plain_texts{$target_id}));
+ last if(!$extract_attributes_regex && ($text_count++ > 1 && $plain_texts{$target_id} && (!$extract_metadata || $metadata{$target_id})));
} elsif ($text_started) {
chomp;
$_ = decode("utf-8", $_, Encode::FB_DEFAULT) . ' ';
@@ -406,6 +453,31 @@
Get sentence boundary information from tagger output rather than from s annotation in structure.xml files.
+=item B<--token-separator>=I<string>
+
+Token separator.
+
+=item B<--(no)comments>
+
+Switch comment printing on or off (default: on).
+
+=item B<--(no)offsets>
+
+Switch offsets printing on or off (default: on).
+
+=item B<--word2vec>
+
+Print output in word2vec (tokenized) one sentence per line format.
+
+=item B<--extract-metadata-regex|-m>
+
+Can be used to extract and print strings from the text headers. Currently only works together with --word2vec option.
+For example:
+
+ korapxml2conllu -m '<textSigle>([^<.]+)' -m '<creatDate>([^<]{7})' --word2vec t/data/wdf19.zip
+
+Will print the document sigle, year and month of the creation date and one sentence per line, separated by tabs.
+
=item B<--help|-h>
Print help information.
@@ -423,11 +495,17 @@
=head1 EXAMPLES
+=head2 Extract to CoNNL-U with posting and div ids:
+
korapxml2conllu -e '(posting/id|div/id)' t/data/wdf19.zip
+=head2 Extract to word2vec input format metadata columns:
+
+ korapxml2conllu --word2vec t/data/wdf19.zip
+
=head1 COPYRIGHT AND LICENSE
-Copyright (C) 2021, L<IDS Mannheim|https://www.ids-mannheim.de/>
+Copyright (C) 2021-2022, L<IDS Mannheim|https://www.ids-mannheim.de/>
Author: Marc Kupietz