Fixed skip and overwrite flags in scripts
Change-Id: Ia43c1ae45f519760c7ef5bf658586be0e39fc038
diff --git a/script/korapxml2krill b/script/korapxml2krill
index 96c3b23..5ec0805 100644
--- a/script/korapxml2krill
+++ b/script/korapxml2krill
@@ -18,6 +18,10 @@
# 2016/02/04
# - renamed to korapxml2krill
# - added Schreibgebrauch support
+#
+# 2016/02/12
+# - fixed foundry skipping
+
sub printhelp {
print <<'EOHELP';
@@ -27,31 +31,31 @@
Call:
korapxml2krill -z --input <directory> --output <filename>
---input|-i <directory> Directory of the document to index
---output|-o <filename> Document name for output (optional),
- Writes to <STDOUT> by default
---overwrite|-w Overwrite files that already exist
---token|-t <foundry>[#<layer>] Define the default tokenization by specifying
- the name of the foundry and optionally the name
- of the layer. Defaults to OpenNLP#tokens.
---skip|-s <foundry>[#<layer>] Skip specific foundries by specifying the name
- or specific layers by defining the name
- with a # in front of the foundry,
- e.g. Mate#Morpho. Alternatively you can skip #ALL.
- Can be set multiple times.
---allow|-a <foundry>#<layer> Allow specific foundries and layers by defining them
- combining the foundry name with a # and the layer name.
---primary|-p Output primary data or not. Defaults to true.
- Can be flagged using --no-primary as well.
---human|-m Represent the data human friendly,
- while the output defaults to JSON
---pretty|-y Pretty print json output
---gzip|-z Compress the output
- (expects a defined output file)
---log|-l The Log4perl log level, defaults to ERROR.
---help|-h Print this document (optional)
+ --input|-i <directory> Directory of the document to index
+ --output|-o <filename> Document name for output (optional),
+ Writes to <STDOUT> by default
+ --overwrite|-w Overwrite files that already exist
+ --token|-t <foundry>[#<layer>] Define the default tokenization by specifying
+ the name of the foundry and optionally the name
+ of the layer. Defaults to OpenNLP#tokens.
+ --skip|-s <foundry>[#<layer>] Skip specific foundries by specifying the name
+ or specific layers by defining the name
+ with a # in front of the foundry,
+ e.g. Mate#Morpho. Alternatively you can skip #ALL.
+ Can be set multiple times.
+ --allow|-a <foundry>#<layer> Allow specific foundries and layers by defining them
+ combining the foundry name with a # and the layer name.
+ --primary|-p Output primary data or not. Defaults to true.
+ Can be flagged using --no-primary as well.
+ --human|-m Represent the data human friendly,
+ while the output defaults to JSON
+ --pretty|-y Pretty print json output
+ --gzip|-z Compress the output
+ (expects a defined output file)
+ --log|-l The Log4perl log level, defaults to ERROR.
+ --help|-h Print this document (optional)
-diewald@ids-mannheim.de, 2016/02/04
+diewald@ids-mannheim.de, 2016/02/12
EOHELP
exit(defined $_[0] ? $_[0] : 0);
@@ -112,8 +116,6 @@
$main::LAST_STOP = $new;
};
-# Call perl script/korapxml2krill WPD/AAA/00001
-
# Create and parse new document
$input =~ s{([^/])$}{$1/};
my $doc = KorAP::XML::Krill->new( path => $input );
@@ -200,7 +202,8 @@
else {
# Add to index file - respect skipping
foreach my $info (@layers) {
- unless ($skip{lc($info->[0]) . '#' . lc($info->[1])}) {
+ # Skip if Foundry or Foundry#Layer should be skipped
+ unless ($skip{lc($info->[0])} || $skip{lc($info->[0]) . '#' . lc($info->[1])}) {
$tokens->add(@$info);
stop_time;
};
diff --git a/script/korapxml2krill_dir b/script/korapxml2krill_dir
index b9db239..7b048cf 100644
--- a/script/korapxml2krill_dir
+++ b/script/korapxml2krill_dir
@@ -13,6 +13,9 @@
#
# 2016/02/04
# - Rename to korapxml2krill_dir
+#
+# 2016/02/12
+# - Support overwrite
sub printhelp {
print <<'EOHELP';
@@ -23,40 +26,43 @@
Call:
korapxml2krill_dir -z --input <directory> --output <directory>
---input|-i <directory> Directory of documents to index
---output|-o <directory> Name of output folder
---token|-t <foundry>[#<layer>] Define the default tokenization by specifying
- the name of the foundry and optionally the name
- of the layer. Defaults to OpenNLP#tokens.
---skip|-s <foundry>[#<layer>] Skip specific foundries by specifying the name
- or specific layers by defining the name
- with a # in front of the foundry,
- e.g. Mate#Morpho. Alternatively you can skip #ALL.
- Can be set multiple times.
---allow|-a <foundry>#<layer> Allow specific foundries and layers by defining them
- combining the foundry name with a # and the layer name.
---primary|-p Output primary data or not. Defaults to true.
- Can be flagged using --no-primary as well.
---human|-m Represent the data human friendly,
- while the output defaults to JSON
---pretty|-y Pretty print json output
---gzip|-z Compress the output
- (expects a defined output file)
---log|-l The Log4perl log level, defaults to ERROR.
---help|-h Print this document (optional)
+ --input|-i <directory> Directory of documents to index
+ --output|-o <directory> Name of output folder
+ --overwrite|-w Overwrite files that already exist
+ --token|-t <foundry>[#<layer>] Define the default tokenization by specifying
+ the name of the foundry and optionally the name
+ of the layer. Defaults to OpenNLP#tokens.
+ --skip|-s <foundry>[#<layer>] Skip specific foundries by specifying the name
+ or specific layers by defining the name
+ with a # in front of the foundry,
+ e.g. Mate#Morpho. Alternatively you can skip #ALL.
+ Can be set multiple times.
+ --allow|-a <foundry>#<layer> Allow specific foundries and layers by defining them
+ combining the foundry name with a # and the layer name.
+ --primary|-p Output primary data or not. Defaults to true.
+ Can be flagged using --no-primary as well.
+ --human|-m Represent the data human friendly,
+ while the output defaults to JSON
+ --pretty|-y Pretty print json output
+ --gzip|-z Compress the output
+ (expects a defined output file)
+ --log|-l The Log4perl log level, defaults to ERROR.
+ --help|-h Print this document (optional)
-diewald@ids-mannheim.de, 2016/02/04
+diewald@ids-mannheim.de, 2016/02/12
EOHELP
exit(defined $_[0] ? $_[0] : 0);
};
-my ($input, $output, $text, $gzip, $log_level, @skip, $token_base, $primary, @allow, $pretty);
+my ($input, $output, $text, $gzip, $log_level, @skip,
+ $token_base, $primary, @allow, $pretty, $overwrite);
GetOptions(
'input|i=s' => \$input,
'output|o=s' => \$output,
'human|m' => \$text,
+ 'overwrite|w' => \$overwrite,
'token|t=s' => \$token_base,
'gzip|z' => \$gzip,
'skip|s=s' => \@skip,
@@ -80,6 +86,7 @@
my $call = 'perl ' . $local . '/korapxml2krill -i ' . $anno . ' -o ' . $output . '/' . $file . '.json';
$call .= '.gz -z' if $gzip;
$call .= ' -m' if $text;
+ $call .= ' -w' if $overwrite;
$call .= ' -t ' . $token_base if $token_base;
$call .= ' -l ' . $log_level if $log_level;
$call .= ' --no-primary ' if $primary;