Nils Diewald | 2db9ad0 | 2013-10-29 19:26:43 +0000 | [diff] [blame] | 1 | #!/usr/bin/env perl |
| 2 | use strict; |
| 3 | use warnings; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 4 | use FindBin; |
| 5 | BEGIN { unshift @INC, "$FindBin::Bin/../lib" }; |
| 6 | use File::Spec::Functions qw/catfile catdir/; |
| 7 | use Getopt::Long qw/GetOptions :config no_auto_abbrev/; |
Nils Diewald | 7364d1f | 2013-11-05 19:26:35 +0000 | [diff] [blame] | 8 | use Benchmark qw/:hireswallclock/; |
| 9 | use IO::Compress::Gzip qw/$GzipError/; |
Nils Diewald | 2db9ad0 | 2013-10-29 19:26:43 +0000 | [diff] [blame] | 10 | use Log::Log4perl; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 11 | use Pod::Usage; |
Akron | 11c8030 | 2016-03-18 19:44:43 +0100 | [diff] [blame] | 12 | use Cache::FastMmap; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 13 | use Directory::Iterator; |
Akron | 93d620e | 2016-02-05 19:40:05 +0100 | [diff] [blame] | 14 | use KorAP::XML::Krill; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 15 | use KorAP::XML::Archive; |
Akron | 93d620e | 2016-02-05 19:40:05 +0100 | [diff] [blame] | 16 | use KorAP::XML::Tokenizer; |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 17 | use KorAP::XML::Batch::File; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 18 | use Parallel::ForkManager; |
Akron | 75ba57d | 2016-03-07 23:36:27 +0100 | [diff] [blame] | 19 | # TODO: use Parallel::Loops |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 20 | # TODO: make output files |
Akron | 93d620e | 2016-02-05 19:40:05 +0100 | [diff] [blame] | 21 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 22 | # CHANGES: |
| 23 | # ---------------------------------------------------------- |
| 24 | # 2013/11/25 |
| 25 | # - Initial release |
| 26 | # |
| 27 | # 2014/10/29 |
| 28 | # - Merges foundry data to create indexer friendly documents |
| 29 | # |
Akron | 93d620e | 2016-02-05 19:40:05 +0100 | [diff] [blame] | 30 | # 2016/02/04 |
| 31 | # - renamed to korapxml2krill |
| 32 | # - added Schreibgebrauch support |
Akron | 069bd71 | 2016-02-12 19:09:06 +0100 | [diff] [blame] | 33 | # |
| 34 | # 2016/02/12 |
| 35 | # - fixed foundry skipping |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 36 | # - Support overwrite in archive processing |
Akron | 150b29e | 2016-02-14 23:06:48 +0100 | [diff] [blame] | 37 | # |
| 38 | # 2016/02/14 |
| 39 | # - Added version information |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 40 | # - Added support for archive files |
| 41 | # |
| 42 | # 2016/02/15 |
| 43 | # - Fixed temporary directory bug |
| 44 | # - Improved skipping before unzipping |
| 45 | # - Added EXPERIMENTAL concurrency support |
| 46 | # |
| 47 | # 2016/02/23 |
| 48 | # - Merge korapxml2krill and korapxml2krill_dir |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 49 | # |
| 50 | # 2016/02/27 |
| 51 | # - Added extract function |
Akron | 35db6e3 | 2016-03-17 22:42:22 +0100 | [diff] [blame] | 52 | # |
| 53 | # 2016/03/17 |
| 54 | # - Added meta switch |
Akron | 11c8030 | 2016-03-18 19:44:43 +0100 | [diff] [blame] | 55 | # |
| 56 | # 2016/03/18 |
| 57 | # - Added meta data caching |
Akron | 2cfe809 | 2016-06-24 17:48:49 +0200 | [diff] [blame] | 58 | # |
Akron | f3f0c94 | 2016-06-27 13:27:14 +0200 | [diff] [blame] | 59 | # 2016/06/27 |
Akron | 2cfe809 | 2016-06-24 17:48:49 +0200 | [diff] [blame] | 60 | # - Added multi archive support |
| 61 | # - Added prefix negation support |
Akron | f3f0c94 | 2016-06-27 13:27:14 +0200 | [diff] [blame] | 62 | # - Added Malt#Dependency support |
Akron | 8b99052 | 2016-07-06 16:45:57 +0200 | [diff] [blame] | 63 | # |
| 64 | # 2016/07/06 |
| 65 | # - Added MDParser#Dependency |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 66 | # |
| 67 | # 2016/10/15 |
Nils Diewald | b3e9ccd | 2016-10-24 15:16:52 +0200 | [diff] [blame^] | 68 | # - Fixed temporary path issue in script |
| 69 | # |
| 70 | # 2016/10/24 |
| 71 | # - Improved Windows support |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 72 | # |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 73 | # ---------------------------------------------------------- |
Akron | 069bd71 | 2016-02-12 19:09:06 +0100 | [diff] [blame] | 74 | |
Nils Diewald | b3e9ccd | 2016-10-24 15:16:52 +0200 | [diff] [blame^] | 75 | our $LAST_CHANGE = '2016/10/24'; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 76 | our $LOCAL = $FindBin::Bin; |
| 77 | our $VERSION_MSG = <<"VERSION"; |
| 78 | Version $KorAP::XML::Krill::VERSION - diewald\@ids-mannheim.de - $LAST_CHANGE |
| 79 | VERSION |
| 80 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 81 | # Parse comand |
| 82 | my $cmd; |
| 83 | our @ARGV; |
| 84 | if ($ARGV[0] && index($ARGV[0], '-') != 0) { |
| 85 | $cmd = shift @ARGV; |
Akron | 150b29e | 2016-02-14 23:06:48 +0100 | [diff] [blame] | 86 | }; |
Akron | 93d620e | 2016-02-05 19:40:05 +0100 | [diff] [blame] | 87 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 88 | my (@skip, @sigle, @anno, @input); |
Akron | 35db6e3 | 2016-03-17 22:42:22 +0100 | [diff] [blame] | 89 | my $text; |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 90 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 91 | # Parse options from the command line |
Nils Diewald | 7364d1f | 2013-11-05 19:26:35 +0000 | [diff] [blame] | 92 | GetOptions( |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 93 | 'input|i=s' => \@input, |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 94 | 'output|o=s' => \(my $output), |
| 95 | 'overwrite|w' => \(my $overwrite), |
Akron | 35db6e3 | 2016-03-17 22:42:22 +0100 | [diff] [blame] | 96 | 'meta|m=s' => \(my $meta), |
Akron | af38698 | 2016-10-12 00:33:25 +0200 | [diff] [blame] | 97 | 'token|t=s' => \(my $token_base = 'OpenNLP#tokens'), |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 98 | 'gzip|z' => \(my $gzip), |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 99 | 'skip|s=s' => \@skip, |
| 100 | 'sigle|sg=s' => \@sigle, |
Akron | 11c8030 | 2016-03-18 19:44:43 +0100 | [diff] [blame] | 101 | 'cache|c=s' => \(my $cache_file = 'korapxml2krill.cache'), |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 102 | 'log|l=s' => \(my $log_level = 'ERROR'), |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 103 | 'anno|a=s' => \@anno, |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 104 | 'primary|p!' => \(my $primary), |
| 105 | 'pretty|y' => \(my $pretty), |
| 106 | 'jobs|j=i' => \(my $jobs = 0), |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 107 | 'cache-size|cs=s' => \(my $cache_size = '50m'), |
| 108 | 'cache-delete|cd!' => \(my $cache_delete = 1), |
| 109 | 'cache-init|ci!' => \(my $cache_init = 1), |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 110 | 'help|h' => sub { |
| 111 | pod2usage( |
| 112 | -sections => 'NAME|SYNOPSIS|ARGUMENTS|OPTIONS', |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 113 | -verbose => 99, |
| 114 | -msg => $VERSION_MSG, |
| 115 | -output => '-' |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 116 | ); |
| 117 | }, |
| 118 | 'version|v' => sub { |
| 119 | pod2usage( |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 120 | -verbose => 0, |
| 121 | -msg => $VERSION_MSG, |
| 122 | -output => '-' |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 123 | ) |
| 124 | } |
Nils Diewald | 7364d1f | 2013-11-05 19:26:35 +0000 | [diff] [blame] | 125 | ); |
| 126 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 127 | my %ERROR_HASH = ( |
| 128 | -sections => 'NAME|SYNOPSIS|ARGUMENTS|OPTIONS', |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 129 | -verbose => 99, |
| 130 | -msg => $VERSION_MSG, |
| 131 | -output => '-', |
| 132 | -exit => 1 |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 133 | ); |
Nils Diewald | 7364d1f | 2013-11-05 19:26:35 +0000 | [diff] [blame] | 134 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 135 | # Input has to be defined |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 136 | pod2usage(%ERROR_HASH) unless @input; |
Nils Diewald | 7364d1f | 2013-11-05 19:26:35 +0000 | [diff] [blame] | 137 | |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 138 | # Gzip has no effect, if no output is given |
| 139 | pod2usage(%ERROR_HASH) if $gzip && !$output; |
Nils Diewald | 7364d1f | 2013-11-05 19:26:35 +0000 | [diff] [blame] | 140 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 141 | # Initialize log4perl object |
Nils Diewald | 7364d1f | 2013-11-05 19:26:35 +0000 | [diff] [blame] | 142 | Log::Log4perl->init({ |
| 143 | 'log4perl.rootLogger' => uc($log_level) . ', STDERR', |
| 144 | 'log4perl.appender.STDERR' => 'Log::Log4perl::Appender::ScreenColoredLevels', |
| 145 | 'log4perl.appender.STDERR.layout' => 'PatternLayout', |
| 146 | 'log4perl.appender.STDERR.layout.ConversionPattern' => '[%r] %F %L %c - %m%n' |
| 147 | }); |
| 148 | |
| 149 | my $log = Log::Log4perl->get_logger('main'); |
| 150 | |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 151 | my %skip; |
| 152 | $skip{lc($_)} = 1 foreach @skip; |
| 153 | |
| 154 | my @layers; |
| 155 | push(@layers, ['Base', 'Sentences']); |
| 156 | push(@layers, ['Base', 'Paragraphs']); |
| 157 | |
| 158 | # Connexor |
| 159 | push(@layers, ['Connexor', 'Morpho']); |
| 160 | push(@layers, ['Connexor', 'Syntax']); |
| 161 | push(@layers, ['Connexor', 'Phrase']); |
| 162 | push(@layers, ['Connexor', 'Sentences']); |
| 163 | |
| 164 | # CoreNLP |
| 165 | push(@layers, ['CoreNLP', 'NamedEntities']); |
| 166 | push(@layers, ['CoreNLP', 'Sentences']); |
| 167 | push(@layers, ['CoreNLP', 'Morpho']); |
| 168 | push(@layers, ['CoreNLP', 'Constituency']); |
| 169 | |
| 170 | # DeReKo |
| 171 | push(@layers, ['DeReKo', 'Structure']); |
| 172 | |
| 173 | # Glemm |
| 174 | push(@layers, ['Glemm', 'Morpho']); |
| 175 | |
| 176 | # Malt |
| 177 | push(@layers, ['Malt', 'Dependency']); |
| 178 | |
| 179 | # MDParser |
| 180 | push(@layers, ['MDParser', 'Dependency']); |
| 181 | |
| 182 | # Mate |
| 183 | push(@layers, ['Mate', 'Morpho']); |
| 184 | push(@layers, ['Mate', 'Dependency']); |
| 185 | |
| 186 | # OpenNLP |
| 187 | push(@layers, ['OpenNLP', 'Morpho']); |
| 188 | push(@layers, ['OpenNLP', 'Sentences']); |
| 189 | |
| 190 | # Schreibgebrauch |
| 191 | push(@layers, ['Sgbr', 'Lemma']); |
| 192 | push(@layers, ['Sgbr', 'Morpho']); |
| 193 | |
| 194 | # TreeTagger |
| 195 | push(@layers, ['TreeTagger', 'Morpho']); |
| 196 | push(@layers, ['TreeTagger', 'Sentences']); |
| 197 | |
| 198 | # XIP |
| 199 | push(@layers, ['XIP', 'Morpho']); |
| 200 | push(@layers, ['XIP', 'Constituency']); |
| 201 | push(@layers, ['XIP', 'Sentences']); |
| 202 | push(@layers, ['XIP', 'Dependency']); |
| 203 | |
| 204 | # Check filters |
| 205 | my @filtered_anno; |
| 206 | if ($skip{'#all'}) { |
| 207 | foreach (@anno) { |
| 208 | push @filtered_anno, [ split('#', $_) ]; |
| 209 | }; |
| 210 | } |
| 211 | |
| 212 | # Add all annotations that are not skipped |
| 213 | else { |
| 214 | # Add to index file - respect skipping |
| 215 | foreach my $info (@layers) { |
| 216 | # Skip if Foundry or Foundry#Layer should be skipped |
| 217 | unless ($skip{lc($info->[0])} || $skip{lc($info->[0]) . '#' . lc($info->[1])}) { |
| 218 | push @filtered_anno, $info; |
| 219 | }; |
| 220 | }; |
| 221 | }; |
| 222 | |
| 223 | # Get tokenization basis |
| 224 | my ($token_base_foundry, $token_base_layer) = split(/#/, $token_base) if ($token_base); |
| 225 | |
| 226 | # TODO: This should not be initialized for batch |
| 227 | my $cache = Cache::FastMmap->new( |
| 228 | share_file => $cache_file, |
| 229 | cache_size => $cache_size, |
Akron | 03b24db | 2016-08-16 20:54:32 +0200 | [diff] [blame] | 230 | init_file => $cache_init |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 231 | ); |
| 232 | |
Akron | 03b24db | 2016-08-16 20:54:32 +0200 | [diff] [blame] | 233 | # Create batch object |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 234 | my $batch_file = KorAP::XML::Batch::File->new( |
Akron | 03b24db | 2016-08-16 20:54:32 +0200 | [diff] [blame] | 235 | cache => $cache, |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 236 | meta_type => $meta, |
| 237 | overwrite => $overwrite, |
Akron | 03b24db | 2016-08-16 20:54:32 +0200 | [diff] [blame] | 238 | foundry => $token_base_foundry, |
| 239 | layer => $token_base_layer, |
| 240 | gzip => $gzip, |
| 241 | log => $log, |
| 242 | primary => $primary, |
| 243 | pretty => $pretty, |
| 244 | anno => \@filtered_anno |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 245 | ); |
| 246 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 247 | |
| 248 | # Get file name based on path information |
| 249 | sub get_file_name ($) { |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 250 | my $i = $input[0]; |
Akron | 3ec4897 | 2016-08-17 23:24:52 +0200 | [diff] [blame] | 251 | if (-d $i) { |
| 252 | $i =~ s![^\/]+$!!; |
| 253 | }; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 254 | my $file = shift; |
Akron | 3ec4897 | 2016-08-17 23:24:52 +0200 | [diff] [blame] | 255 | |
| 256 | # Remove temp dir fragments |
Akron | 6255760 | 2016-06-27 14:10:13 +0200 | [diff] [blame] | 257 | $file =~ s!^/?tmp/[^/]+!!; |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 258 | $file =~ s/^?\/?$i//; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 259 | $file =~ tr/\//-/; |
| 260 | $file =~ s{^-+}{}; |
| 261 | return $file; |
Nils Diewald | 59094f2 | 2014-11-05 18:20:50 +0000 | [diff] [blame] | 262 | }; |
| 263 | |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 264 | # Convert sigle to path construct |
| 265 | s!^\s*([^_]+?)_([^\.]+?)\.(.+?)\s*$!$1/$2/$3! foreach @sigle; |
| 266 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 267 | if ($cmd) { |
| 268 | if ($output && (!-e $output || !-d $output)) { |
| 269 | print "Directory '$output' does not exist.\n\n"; |
| 270 | exit(0); |
| 271 | }; |
| 272 | }; |
| 273 | |
| 274 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 275 | # Process a single file |
| 276 | unless ($cmd) { |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 277 | my $input = $input[0]; |
Nils Diewald | 59094f2 | 2014-11-05 18:20:50 +0000 | [diff] [blame] | 278 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 279 | BEGIN { |
| 280 | $main::TIME = Benchmark->new; |
| 281 | $main::LAST_STOP = Benchmark->new; |
| 282 | }; |
| 283 | |
| 284 | sub stop_time { |
| 285 | my $new = Benchmark->new; |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 286 | $log->info( |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 287 | 'The code took: '. |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 288 | timestr(timediff($new, $main::LAST_STOP)) . |
| 289 | ' (overall: ' . timestr(timediff($new, $main::TIME)) . ')' |
| 290 | ); |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 291 | $main::LAST_STOP = $new; |
| 292 | }; |
| 293 | |
| 294 | # Create and parse new document |
| 295 | $input =~ s{([^/])$}{$1/}; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 296 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 297 | # Process file |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 298 | $batch_file->process($input, $output); |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 299 | |
Akron | 11c8030 | 2016-03-18 19:44:43 +0100 | [diff] [blame] | 300 | # Delete cache file |
| 301 | unlink($cache_file) if $cache_delete; |
| 302 | |
Akron | 5f51d42 | 2016-08-16 16:26:43 +0200 | [diff] [blame] | 303 | stop_time; |
Nils Diewald | 7364d1f | 2013-11-05 19:26:35 +0000 | [diff] [blame] | 304 | } |
Nils Diewald | 59094f2 | 2014-11-05 18:20:50 +0000 | [diff] [blame] | 305 | |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 306 | # Extract XML files |
| 307 | elsif ($cmd eq 'extract') { |
| 308 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 309 | # Create new archive object |
Akron | b0c88db | 2016-06-29 16:33:18 +0200 | [diff] [blame] | 310 | if (-f($input[0]) && (my $archive = KorAP::XML::Archive->new($input[0]))) { |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 311 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 312 | # Check zip capabilities |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 313 | unless ($archive->test_unzip) { |
| 314 | print "Unzip is not installed or incompatible.\n\n"; |
| 315 | exit(1); |
| 316 | }; |
| 317 | |
Akron | b0c88db | 2016-06-29 16:33:18 +0200 | [diff] [blame] | 318 | # Add further annotation archived |
| 319 | $archive->attach($_) foreach @input; |
| 320 | |
Akron | 651cb8d | 2016-08-16 21:44:49 +0200 | [diff] [blame] | 321 | my $prefix = 1; |
| 322 | |
Akron | 03b24db | 2016-08-16 20:54:32 +0200 | [diff] [blame] | 323 | # No sigles given |
| 324 | unless (@sigle) { |
| 325 | |
| 326 | # Get files |
| 327 | foreach ($archive->list_texts) { |
| 328 | |
| 329 | # Split path information |
Akron | 651cb8d | 2016-08-16 21:44:49 +0200 | [diff] [blame] | 330 | ($prefix, my ($corpus, $doc, $text)) = $archive->split_path($_); |
Akron | 03b24db | 2016-08-16 20:54:32 +0200 | [diff] [blame] | 331 | |
| 332 | # TODO: Make this OS independent |
| 333 | push @sigle, join '/', $corpus, $doc, $text; |
| 334 | }; |
| 335 | }; |
| 336 | |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 337 | # Iterate over all given sigles and extract |
| 338 | foreach (@sigle) { |
| 339 | print "$_ "; |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 340 | |
Akron | 03b24db | 2016-08-16 20:54:32 +0200 | [diff] [blame] | 341 | # TODO: Make this OS independent |
Akron | 651cb8d | 2016-08-16 21:44:49 +0200 | [diff] [blame] | 342 | print '' . ( |
| 343 | $archive->extract( |
| 344 | ($prefix ? './' : '') . $_, $output |
| 345 | ) ? '' : 'not ' |
| 346 | ); |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 347 | print "extracted.\n"; |
| 348 | }; |
| 349 | |
| 350 | print "\n"; |
| 351 | exit(1); |
Akron | b0c88db | 2016-06-29 16:33:18 +0200 | [diff] [blame] | 352 | } |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 353 | |
| 354 | # Can't create archive object |
Akron | b0c88db | 2016-06-29 16:33:18 +0200 | [diff] [blame] | 355 | else { |
| 356 | $log->error('Unable to extract from primary archive ' . $input[0]); |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 357 | }; |
| 358 | } |
| 359 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 360 | # Process an archive |
| 361 | elsif ($cmd eq 'archive') { |
Nils Diewald | 2db9ad0 | 2013-10-29 19:26:43 +0000 | [diff] [blame] | 362 | |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 363 | # TODO: Support sigles |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 364 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 365 | # Zero means: everything runs in the parent process |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 366 | my $pool = Parallel::ForkManager->new($jobs); |
| 367 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 368 | my $count = 0; # Texts to process |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 369 | my $iter = 1; # Current text in process |
| 370 | |
| 371 | # Report on fork message |
| 372 | $pool->run_on_finish ( |
| 373 | sub { |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 374 | my ($pid, $code) = @_; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 375 | my $data = pop; |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 376 | |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 377 | print 'Convert ['. ($jobs > 0 ? "\$$pid:" : '') . |
Akron | 651cb8d | 2016-08-16 21:44:49 +0200 | [diff] [blame] | 378 | ($iter++) . "/$count]" . |
| 379 | ($code ? " $code" : '') . |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 380 | ' ' . $data->[0] . "\n"; |
| 381 | $data->[1] = undef if $data->[1]; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 382 | } |
| 383 | ); |
| 384 | |
| 385 | my $t; |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 386 | my $temp; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 387 | print "Reading data ...\n"; |
| 388 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 389 | # unless (Cache::FastMmap->new( |
| 390 | # share_file => $cache_file, |
| 391 | # cache_size => $cache_size, |
| 392 | # init_file => $cache_init |
| 393 | # )) { |
| 394 | # print "Unable to intialize cache '$cache_file'\n\n"; |
| 395 | # exit(1); |
| 396 | # }; |
Akron | 11c8030 | 2016-03-18 19:44:43 +0100 | [diff] [blame] | 397 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 398 | # Input is a directory |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 399 | if (-d $input[0]) { |
| 400 | my $it = Directory::Iterator->new($input[0]); |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 401 | my @dirs; |
| 402 | my $dir; |
| 403 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 404 | # Todo: Make a DO WHILE |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 405 | while (1) { |
| 406 | if (!$it->is_directory && ($dir = $it->get) && $dir =~ s{/data\.xml$}{}) { |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 407 | push @dirs, $dir; |
| 408 | $it->prune; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 409 | }; |
| 410 | last unless $it->next; |
| 411 | }; |
| 412 | |
| 413 | print "Start processing ...\n"; |
| 414 | $t = Benchmark->new; |
| 415 | $count = scalar @dirs; |
| 416 | |
| 417 | DIRECTORY_LOOP: |
| 418 | for (my $i = 0; $i < $count; $i++) { |
| 419 | |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 420 | my $filename = catfile( |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 421 | $output, |
| 422 | get_file_name($dirs[$i]) . '.json' . ($gzip ? '.gz' : '') |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 423 | ); |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 424 | |
| 425 | # Get the next fork |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 426 | $pool->start and next DIRECTORY_LOOP; |
Akron | 3ec4897 | 2016-08-17 23:24:52 +0200 | [diff] [blame] | 427 | |
| 428 | if ($batch_file->process($dirs[$i] => $filename)) { |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 429 | $pool->finish(0, ["Processed " . $filename]); |
Akron | 3ec4897 | 2016-08-17 23:24:52 +0200 | [diff] [blame] | 430 | } |
| 431 | else { |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 432 | $pool->finish(1, ["Unable to process " . $dirs[$i]]); |
Akron | 3ec4897 | 2016-08-17 23:24:52 +0200 | [diff] [blame] | 433 | }; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 434 | }; |
| 435 | } |
| 436 | |
| 437 | # Input is a file |
Akron | 29866ac | 2016-06-24 16:40:47 +0200 | [diff] [blame] | 438 | elsif (-f($input[0]) && (my $archive = KorAP::XML::Archive->new($input[0]))) { |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 439 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 440 | unless ($archive->test_unzip) { |
| 441 | print "Unzip is not installed or incompatible.\n\n"; |
| 442 | exit(1); |
| 443 | }; |
| 444 | |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 445 | # Add further annotation archived |
Akron | 29866ac | 2016-06-24 16:40:47 +0200 | [diff] [blame] | 446 | $archive->attach($_) foreach @input; |
Akron | 08385f6 | 2016-03-22 20:37:04 +0100 | [diff] [blame] | 447 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 448 | print "Start processing ...\n"; |
| 449 | $t = Benchmark->new; |
| 450 | my @dirs = $archive->list_texts; |
| 451 | $count = scalar @dirs; |
| 452 | |
| 453 | ARCHIVE_LOOP: |
| 454 | for (my $i = 0; $i < $count; $i++) { |
| 455 | |
| 456 | # Split path information |
| 457 | my ($prefix, $corpus, $doc, $text) = $archive->split_path($dirs[$i]); |
| 458 | |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 459 | my $filename = catfile( |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 460 | $output, |
| 461 | get_file_name( |
| 462 | catfile($corpus, $doc, $text) |
| 463 | . '.json' . ($gzip ? '.gz' : '') |
| 464 | ) |
Akron | e1dbc38 | 2016-07-08 22:24:52 +0200 | [diff] [blame] | 465 | ); |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 466 | |
| 467 | # Get the next fork |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 468 | $pool->start and next ARCHIVE_LOOP; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 469 | |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 470 | # Create temporary file |
| 471 | $temp = File::Temp->newdir; |
| 472 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 473 | # Extract from archive |
| 474 | if ($archive->extract($dirs[$i], $temp)) { |
| 475 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 476 | # Create corpus directory |
| 477 | my $input = catdir("$temp", $corpus); |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 478 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 479 | # Temporary directory |
| 480 | my $dir = catdir($input, $doc, $text); |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 481 | |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 482 | # Write file |
| 483 | if ($batch_file->process($dir => $filename)) { |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 484 | # Delete temporary file |
| 485 | $pool->finish(0, ["Processed " . $filename, $temp]); |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 486 | } |
| 487 | else { |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 488 | # Delete temporary file |
| 489 | $pool->finish(1, ["Unable to process " . $dir, $temp]); |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 490 | }; |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 491 | } |
Akron | 7d4cdd8 | 2016-08-17 21:39:45 +0200 | [diff] [blame] | 492 | |
| 493 | # Unable to extract |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 494 | else { |
Akron | 4c0cf31 | 2016-10-15 16:42:09 +0200 | [diff] [blame] | 495 | $pool->finish(1, ["Unable to extract " . $dirs[$i], $temp]); |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 496 | }; |
| 497 | }; |
| 498 | } |
| 499 | |
| 500 | else { |
| 501 | print "Input is neither a directory nor an archive.\n\n"; |
| 502 | }; |
| 503 | |
| 504 | $pool->wait_all_children; |
| 505 | |
Akron | 11c8030 | 2016-03-18 19:44:43 +0100 | [diff] [blame] | 506 | # Delete cache file |
| 507 | unlink($cache_file) if $cache_delete; |
| 508 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 509 | print "Done.\n"; |
| 510 | print timestr(timediff(Benchmark->new, $t))."\n\n"; |
| 511 | } |
| 512 | |
| 513 | # Unknown command |
| 514 | else { |
| 515 | warn "Unknown command '$cmd'.\n\n"; |
| 516 | pod2usage(%ERROR_HASH); |
| 517 | } |
Nils Diewald | 2db9ad0 | 2013-10-29 19:26:43 +0000 | [diff] [blame] | 518 | |
| 519 | __END__ |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 520 | |
| 521 | =pod |
| 522 | |
| 523 | =encoding utf8 |
| 524 | |
| 525 | =head1 NAME |
| 526 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 527 | korapxml2krill - Merge KorapXML data and create Krill documents |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 528 | |
| 529 | |
| 530 | =head1 SYNOPSIS |
| 531 | |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 532 | $ korapxml2krill -z --input <directory> --output <filename> |
| 533 | $ korapxml2krill archive -z --input <directory> --output <directory> |
| 534 | $ korapxml2krill extract --input <directory> --output <filename> --sigle <SIGLE> |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 535 | |
| 536 | |
| 537 | =head1 DESCRIPTION |
| 538 | |
| 539 | L<KorAP::XML::Krill> is a library to convert KorAP-XML documents to files |
| 540 | compatible with the L<Krill|https://github.com/KorAP/Krill> indexer. |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 541 | The C<korapxml2krill> command line tool is a simple wrapper to the library. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 542 | |
| 543 | |
| 544 | =head1 INSTALLATION |
| 545 | |
| 546 | The preferred way to install L<KorAP::XML::Krill> is to use L<cpanm|App::cpanminus>. |
| 547 | |
Akron | af38698 | 2016-10-12 00:33:25 +0200 | [diff] [blame] | 548 | $ cpanm https://github.com/KorAP/KorAP-XML-Krill.git |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 549 | |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 550 | In case everything went well, the C<korapxml2krill> tool will |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 551 | be available on your command line immediately. |
Akron | 7438151 | 2016-10-14 11:56:22 +0200 | [diff] [blame] | 552 | Minimum requirement for L<KorAP::XML::Krill> is Perl 5.14. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 553 | |
| 554 | =head1 ARGUMENTS |
| 555 | |
| 556 | =over 2 |
| 557 | |
| 558 | =item B<archive> |
| 559 | |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 560 | Process an archive as a Zip-file or a folder of KorAP-XML documents. |
| 561 | |
| 562 | =item B<extract> |
| 563 | |
| 564 | Extract KorAP-XML files from a Zip-file. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 565 | |
| 566 | =back |
| 567 | |
| 568 | |
| 569 | =head1 OPTIONS |
| 570 | |
| 571 | =over 2 |
| 572 | |
Akron | 2cfe809 | 2016-06-24 17:48:49 +0200 | [diff] [blame] | 573 | =item B<--input|-i> <directory|file|files> |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 574 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 575 | Directory or archive file of documents to convert. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 576 | |
Akron | 0c3e375 | 2016-06-28 15:55:53 +0200 | [diff] [blame] | 577 | Archiving supports multiple input archives with the constraint, |
Akron | 2cfe809 | 2016-06-24 17:48:49 +0200 | [diff] [blame] | 578 | that the first archive listed contains all primary data files |
| 579 | and all meta data files. |
Akron | 2cfe809 | 2016-06-24 17:48:49 +0200 | [diff] [blame] | 580 | |
| 581 | -i file/news.zip -i file/news.malt.zip -i #file/news.tt.zip |
| 582 | |
Akron | 0c3e375 | 2016-06-28 15:55:53 +0200 | [diff] [blame] | 583 | (The directory structure follows the base directory format, |
| 584 | that may include a C<.> root folder. |
| 585 | In this case further archives lacking a C<.> root folder |
| 586 | need to be passed with a hash sign in front of the archive's name.) |
Akron | 2cfe809 | 2016-06-24 17:48:49 +0200 | [diff] [blame] | 587 | |
Akron | 651cb8d | 2016-08-16 21:44:49 +0200 | [diff] [blame] | 588 | B<The root folder switch is experimental and may vanish in future versions.> |
| 589 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 590 | =item B<--output|-o> <directory|file> |
| 591 | |
| 592 | Output folder for archive processing or |
| 593 | document name for single output (optional), |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 594 | writes to C<STDOUT> by default |
| 595 | (in case C<output> is not mandatory due to further options). |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 596 | |
| 597 | =item B<--overwrite|-w> |
| 598 | |
| 599 | Overwrite files that already exist. |
| 600 | |
| 601 | =item B<--token|-t> <foundry>[#<file>] |
| 602 | |
| 603 | Define the default tokenization by specifying |
| 604 | the name of the foundry and optionally the name |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 605 | of the layer-file. Defaults to C<OpenNLP#tokens>. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 606 | |
| 607 | =item B<--skip|-s> <foundry>[#<layer>] |
| 608 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 609 | Skip specific annotations by specifying the foundry |
| 610 | (and optionally the layer with a C<#>-prefix), |
| 611 | e.g. C<Mate> or C<Mate#Morpho>. Alternatively you can skip C<#ALL>. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 612 | Can be set multiple times. |
| 613 | |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 614 | =item B<--anno|-a> <foundry>#<layer> |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 615 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 616 | Convert specific annotations by specifying the foundry |
| 617 | (and optionally the layer with a C<#>-prefix), |
| 618 | e.g. C<Mate> or C<Mate#Morpho>. |
| 619 | Can be set multiple times. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 620 | |
| 621 | =item B<--primary|-p> |
| 622 | |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 623 | Output primary data or not. Defaults to C<true>. |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 624 | Can be flagged using C<--no-primary> as well. |
| 625 | This is I<deprecated>. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 626 | |
| 627 | =item B<--jobs|-j> |
| 628 | |
| 629 | Define the number of concurrent jobs in seperated forks |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 630 | for archive processing. |
Akron | 11c8030 | 2016-03-18 19:44:43 +0100 | [diff] [blame] | 631 | Defaults to C<0> (everything runs in a single process). |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 632 | This is I<experimental>. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 633 | |
Akron | 35db6e3 | 2016-03-17 22:42:22 +0100 | [diff] [blame] | 634 | =item B<--meta|-m> |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 635 | |
Akron | 35db6e3 | 2016-03-17 22:42:22 +0100 | [diff] [blame] | 636 | Define the metadata parser to use. Defaults to C<I5>. |
| 637 | Metadata parsers can be defined in the C<KorAP::XML::Meta> namespace. |
| 638 | This is I<experimental>. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 639 | |
| 640 | =item B<--pretty|-y> |
| 641 | |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 642 | Pretty print JSON output. Defaults to C<false>. |
Akron | 35db6e3 | 2016-03-17 22:42:22 +0100 | [diff] [blame] | 643 | This is I<deprecated>. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 644 | |
| 645 | =item B<--gzip|-z> |
| 646 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 647 | Compress the output. |
| 648 | Expects a defined C<output> file in single processing. |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 649 | |
Akron | 11c8030 | 2016-03-18 19:44:43 +0100 | [diff] [blame] | 650 | =item B<--cache|-c> |
| 651 | |
| 652 | File to mmap a cache (using L<Cache::FastMmap>). |
| 653 | Defaults to C<korapxml2krill.cache> in the calling directory. |
| 654 | |
| 655 | =item B<--cache-size|-cs> |
| 656 | |
| 657 | Size of the cache. Defaults to C<50m>. |
| 658 | |
| 659 | =item B<--cache-init|-ci> |
| 660 | |
| 661 | Initialize cache file. |
| 662 | Can be flagged using C<--no-cache-init> as well. |
| 663 | Defaults to C<true>. |
| 664 | |
| 665 | =item B<--cache-delete|-cd> |
| 666 | |
| 667 | Delete cache file after processing. |
| 668 | Can be flagged using C<--no-cache-delete> as well. |
| 669 | Defaults to C<true>. |
| 670 | |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 671 | =item B<--sigle|-sg> |
| 672 | |
| 673 | Extract the given text sigles. |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 674 | Can be set multiple times. |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 675 | I<Currently only supported on C<extract>.> |
Akron | b0c88db | 2016-06-29 16:33:18 +0200 | [diff] [blame] | 676 | Sigles have the structure C<Corpus>/C<Document>/C<Text>. |
Akron | e10ad32 | 2016-02-27 10:54:26 +0100 | [diff] [blame] | 677 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 678 | =item B<--log|-l> |
| 679 | |
| 680 | The L<Log4perl> log level, defaults to C<ERROR>. |
| 681 | |
| 682 | =item B<--help|-h> |
| 683 | |
| 684 | Print this document. |
| 685 | |
| 686 | =item B<--version|-v> |
| 687 | |
| 688 | Print version information. |
| 689 | |
| 690 | =back |
| 691 | |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 692 | =head1 ANNOTATION SUPPORT |
| 693 | |
| 694 | L<KorAP::XML::Krill> has built-in importer for some annotation foundries and layers |
| 695 | developed in the KorAP project that are part of the KorAP preprocessing pipeline. |
| 696 | The base foundry with paragraphs, sentences, and the text element are mandatory for |
| 697 | L<Krill|https://github.com/KorAP/Krill>. |
| 698 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 699 | =over 2 |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 700 | |
| 701 | =item B<Base> |
| 702 | |
| 703 | =over 4 |
| 704 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 705 | =item #Paragraphs |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 706 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 707 | =item #Sentences |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 708 | |
| 709 | =back |
| 710 | |
| 711 | =item B<Connexor> |
| 712 | |
| 713 | =over 4 |
| 714 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 715 | =item #Morpho |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 716 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 717 | =item #Phrase |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 718 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 719 | =item #Sentences |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 720 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 721 | =item #Syntax |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 722 | |
| 723 | =back |
| 724 | |
| 725 | =item B<CoreNLP> |
| 726 | |
| 727 | =over 4 |
| 728 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 729 | =item #Constituency |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 730 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 731 | =item #Morpho |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 732 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 733 | =item #NamedEntities |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 734 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 735 | =item #Sentences |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 736 | |
| 737 | =back |
| 738 | |
| 739 | =item B<DeReKo> |
| 740 | |
| 741 | =over 4 |
| 742 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 743 | =item #Structure |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 744 | |
| 745 | =back |
| 746 | |
| 747 | =item B<Glemm> |
| 748 | |
| 749 | =over 4 |
| 750 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 751 | =item #Morpho |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 752 | |
| 753 | =back |
| 754 | |
| 755 | =item B<Mate> |
| 756 | |
| 757 | =over 4 |
| 758 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 759 | =item #Dependency |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 760 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 761 | =item #Morpho |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 762 | |
| 763 | =back |
| 764 | |
| 765 | =item B<OpenNLP> |
| 766 | |
| 767 | =over 4 |
| 768 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 769 | =item #Morpho |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 770 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 771 | =item #Sentences |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 772 | |
| 773 | =back |
| 774 | |
| 775 | =item B<Sgbr> |
| 776 | |
| 777 | =over 4 |
| 778 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 779 | =item #Lemma |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 780 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 781 | =item #Morpho |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 782 | |
| 783 | =back |
| 784 | |
| 785 | =item B<TreeTagger> |
| 786 | |
| 787 | =over 4 |
| 788 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 789 | =item #Morpho |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 790 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 791 | =item #Sentences |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 792 | |
| 793 | =back |
| 794 | |
| 795 | =item B<XIP> |
| 796 | |
| 797 | =over 4 |
| 798 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 799 | =item #Constituency |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 800 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 801 | =item #Morpho |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 802 | |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 803 | =item #Sentences |
Akron | c13a170 | 2016-03-15 19:33:14 +0100 | [diff] [blame] | 804 | |
| 805 | =back |
| 806 | |
| 807 | =back |
| 808 | |
| 809 | More importers are in preparation. |
| 810 | New annotation importers can be defined in the C<KorAP::XML::Annotation> namespace. |
| 811 | See the built-in annotation importers as examples. |
| 812 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 813 | =head1 AVAILABILITY |
| 814 | |
| 815 | https://github.com/KorAP/KorAP-XML-Krill |
| 816 | |
| 817 | |
| 818 | =head1 COPYRIGHT AND LICENSE |
| 819 | |
| 820 | Copyright (C) 2015-2016, L<IDS Mannheim|http://www.ids-mannheim.de/> |
Akron | f7ad89e | 2016-03-16 18:22:47 +0100 | [diff] [blame] | 821 | |
Akron | 941c1a6 | 2016-02-23 17:41:41 +0100 | [diff] [blame] | 822 | Author: L<Nils Diewald|http://nils-diewald.de/> |
| 823 | |
| 824 | L<KorAP::XML::Krill> is developed as part of the L<KorAP|http://korap.ids-mannheim.de/> |
| 825 | Corpus Analysis Platform at the |
| 826 | L<Institute for the German Language (IDS)|http://ids-mannheim.de/>, |
| 827 | member of the |
| 828 | L<Leibniz-Gemeinschaft|http://www.leibniz-gemeinschaft.de/en/about-us/leibniz-competition/projekte-2011/2011-funding-line-2/>. |
| 829 | |
| 830 | This program is free software published under the |
| 831 | L<BSD-2 License|https://raw.githubusercontent.com/KorAP/KorAP-XML-Krill/master/LICENSE>. |
| 832 | |
| 833 | =cut |