Introduce --inline-tokens parameter
Change-Id: I11f82ff145c4278d720563703f23c0c87bd84340
diff --git a/Changes b/Changes
index a645256..53421c2 100644
--- a/Changes
+++ b/Changes
@@ -5,6 +5,7 @@
- resources defined in Makefile
- fixed possible IO deadlock with KorAP tokenizer
- Simplified debugging by combining with X::C::T line numbers
+ - Support inline-tokens parameter
0.03 2021-01-12
- Update KorAP-Tokenizer to released 2.0 version
diff --git a/Readme.pod b/Readme.pod
index 209df07..1689eb4 100644
--- a/Readme.pod
+++ b/Readme.pod
@@ -112,6 +112,14 @@
that will take an I<Aggressive> and a I<conservative>
approach.
+=item B<--inline-tokens> <foundry>#[<file>]
+
+Define the foundry and file (without extension)
+to store inline token information in.
+If L</KORAPXMLTEI_INLINE> is set, this will contain
+annotations as well.
+Defaults to C<tokens> and C<morpho>.
+
=item B<--use-tokenizer-sentence-splits|-s>
Replace existing with, or add new, sentence boundary information
diff --git a/script/tei2korapxml b/script/tei2korapxml
index d4ccb5b..eb97909 100755
--- a/script/tei2korapxml
+++ b/script/tei2korapxml
@@ -48,6 +48,7 @@
'tokenizer-korap|tk' => \(my $tokenizer_korap), # use KorAP-tokenizer
'tokenizer-internal|ti' => \(my $tokenizer_intern), # use intern tokenization (default = no)
'use-tokenizer-sentence-splits|s' => (\my $use_tokenizer_sentence_splits), # use KorAP tokenizer to split s (default=no)
+ 'inline-tokens=s' => \(my $inline_tokens = 'tokens#morpho'),
'log|l=s' => \(my $log_level = 'notice'),
'help|h' => sub {
pod2usage(
@@ -125,9 +126,13 @@
# (= their names and byte offsets in $_data)
## TODO: optional (different annotation tools can produce more zip-files for feeding into KorAP-XML-Krill)
my $_TOKENS_PROC = 1; # on/off: processing of ${_TOKEN_TAG}'s (default: 1)
-my $_tokens_dir = "tokens"; # name of directory containing the $_tokens_file
-my $_tokens_file = "morpho.xml"; # name of file containing all ${_TOKEN_TAG}'s related information (=their byte offsets in $_data)
- # - evtl. with additional inline annotations
+
+
+# Name of the directory and the file containing all inline token informations
+# i.e. tokens of the $_TOKENS_TAG, if $_TOKENS_PROC is set
+my ($_tokens_dir, $_tokens_file) = split '#', $inline_tokens . '#morpho';
+$_tokens_file .= '.xml';
+
my $_TOKENS_TAG = "w"; # name of tag containing all information stored in $_tokens_file
# Handling inline annotations (inside $_TOKENS_TAG)
@@ -892,6 +897,14 @@
that will take an I<Aggressive> and a I<conservative>
approach.
+=item B<--inline-tokens> <foundry>#[<file>]
+
+Define the foundry and file (without extension)
+to store inline token information in.
+If L</KORAPXMLTEI_INLINE> is set, this will contain
+annotations as well.
+Defaults to C<tokens> and C<morpho>.
+
=item B<--use-tokenizer-sentence-splits|-s>
Replace existing with, or add new, sentence boundary information
diff --git a/t/script.t b/t/script.t
index 773bb60..0ac27d9 100644
--- a/t/script.t
+++ b/t/script.t
@@ -415,6 +415,51 @@
;
};
+subtest 'Check Inline annotations with defined foundry and folder' => sub {
+ # Load example file
+ my $file = catfile($f, 'data', 'goe_sample_tagged.i5.xml');
+
+ my $t = test_tei2korapxml(
+ file => $file,
+ env => 'KORAPXMLTEI_INLINE=1',
+ tmp => 'script_tagged',
+ param => '--inline-tokens=myfoundry#myfile'
+ )
+ ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
+
+ ->file_exists_not('GOE/AGA/00000/tokens/morpho.xml', 'Morpho not generated')
+
+ # Check zip using xml loy
+ ->unzip_xml('GOE/AGA/00000/myfoundry/myfile.xml')
+
+ ->attr_is('layer', 'docid', 'GOE_AGA.00000')
+ ->attr_is('spanList span:nth-child(1)', 'id', 's0')
+ ->attr_is('spanList span:nth-child(1)', 'from', '75')
+ ->attr_is('spanList span:nth-child(1)', 'to', '81')
+ ->attr_is('spanList span:nth-child(1)', 'l', '7')
+ ;
+
+ $t = test_tei2korapxml(
+ file => $file,
+ env => 'KORAPXMLTEI_INLINE=1',
+ tmp => 'script_tagged',
+ param => '--inline-tokens=myfoundry'
+ )
+ ->stderr_like(qr!tei2korapxml: .*? text_id=GOE_AGA\.00000!)
+
+ ->file_exists_not('GOE/AGA/00000/tokens/morpho.xml', 'Morpho not generated')
+
+ # Check zip using xml loy
+ ->unzip_xml('GOE/AGA/00000/myfoundry/morpho.xml')
+
+ ->attr_is('layer', 'docid', 'GOE_AGA.00000')
+ ->attr_is('spanList span:nth-child(1)', 'id', 's0')
+ ->attr_is('spanList span:nth-child(1)', 'from', '75')
+ ->attr_is('spanList span:nth-child(1)', 'to', '81')
+ ->attr_is('spanList span:nth-child(1)', 'l', '7')
+ ;
+};
+
subtest 'Check Inline annotations with untagged file' => sub {
# Load example file