Move inline parser to separate class
Change-Id: I835acf7a234b385dc3d2a4e52c6ada61dbe087db
diff --git a/Changes b/Changes
index 581cbc8..9a8ad95 100644
--- a/Changes
+++ b/Changes
@@ -5,6 +5,7 @@
- Introduce --skip-inline-tokens parameter
- Minor cleanups and improvements
- Introduce --skip-inline-tags parameter
+ - Introduce KorAP::XML::TEI::Inline class
1.00 2021-02-18 Release
- -s option added that uses sentence boundaries
diff --git a/lib/KorAP/XML/TEI/Annotations/Collector.pm b/lib/KorAP/XML/TEI/Annotations/Collector.pm
index 510d339..887bbb0 100644
--- a/lib/KorAP/XML/TEI/Annotations/Collector.pm
+++ b/lib/KorAP/XML/TEI/Annotations/Collector.pm
@@ -43,10 +43,10 @@
# Correct tokens
# TODO:
# Check if this is also necessary for structures
- if ($param != STRUCTURE) {
+ if ($param != STRUCTURE && !$self->empty) {
# correct last from-value (if the 'second to last'
# from-value refers to an s-tag, then the last from-value
- # is one to big - see retr_info())
+ # is one to big - see _descend())
my $last_token = $_[0]->[$#{$_[0]}];
if ($last_token->from == $last_token->to + 1) {
# TODO:
diff --git a/lib/KorAP/XML/TEI/Inline.pm b/lib/KorAP/XML/TEI/Inline.pm
new file mode 100644
index 0000000..d1eafdc
--- /dev/null
+++ b/lib/KorAP/XML/TEI/Inline.pm
@@ -0,0 +1,450 @@
+package KorAP::XML::TEI::Inline;
+use strict;
+use warnings;
+use Log::Any '$log';
+use XML::CompactTree::XS;
+use XML::LibXML::Reader;
+
+use KorAP::XML::TEI::Data;
+use KorAP::XML::TEI::Annotations::Collector;
+
+# Parsing of inline annotations in i5 files
+
+# name of the tag containing all information stored in $_tokens_file
+our $_TOKENS_TAG = 'w';
+
+# TODO:
+# Replace whitespace handling with Bit::Vector
+
+use constant {
+ # XCT_LINE_NUMBERS is only needed for debugging
+ # (see XML::CompactTree::XS)
+ XCT_PARAM => (
+ XCT_DOCUMENT_ROOT
+ | XCT_IGNORE_COMMENTS
+ | XCT_ATTRIBUTE_ARRAY
+ | ($ENV{KORAPXMLTEI_DEBUG} ? XCT_LINE_NUMBERS : 0)
+ ),
+
+ # Set to 1 for minimal more debug output (no need to be parametrized)
+ DEBUG => $ENV{KORAPXMLTEI_DEBUG} // 0,
+
+ # Array constants
+ ADD_ONE => 0,
+ WS => 1,
+ TEXT_ID => 2,
+ DATA => 3,
+ TOKENS => 5,
+ STRUCTURES => 6,
+ SKIP_INLINE_TAGS => 7,
+ SKIP_INLINE_TOKENS => 8
+};
+
+
+# Constructor
+sub new {
+ my ($class, $skip_inline_tokens, $skip_inline_tags) = @_;
+
+ my @self = ();
+
+ # variables for handling ~ whitespace related issue ~
+ # (it is sometimes necessary, to correct the from-values for some tags)
+ $self[ADD_ONE] = 0;
+
+ # hash for indices of whitespace-nodes
+ # (needed to recorrect from-values)
+ # IDEA:
+ # when closing element, check if it's from-index minus 1 refers to a whitespace-node
+ # (means: 'from-index - 1' is a key in %ws).
+ # if this is _not_ the case, then the from-value is one
+ # to high => correct it by substracting 1
+ $self[WS] = {};
+
+ # Initialize data collector
+ $self[DATA] = KorAP::XML::TEI::Data->new;
+
+ # Initialize token collector
+ $self[TOKENS] = KorAP::XML::TEI::Annotations::Collector->new;
+
+ # Initialize structure collector
+ $self[STRUCTURES] = KorAP::XML::TEI::Annotations::Collector->new;
+ $self[SKIP_INLINE_TOKENS] = $skip_inline_tokens // undef;
+ $self[SKIP_INLINE_TAGS] = $skip_inline_tags // {};
+
+ bless \@self, $class;
+};
+
+
+# Parse inline data
+sub parse {
+ my ($self, $text_id_esc, $text_buffer_ref) = @_;
+
+ $self->[TEXT_ID] = $text_id_esc;
+
+ # Whitespace related issue
+ $self->[ADD_ONE] = 0;
+ $self->[WS] = {};
+
+ # Reset all collectors
+ $self->[DATA]->reset;
+ $self->[STRUCTURES]->reset;
+ $self->[TOKENS]->reset;
+
+ # Create XML::LibXML::Reader
+ my $reader = XML::LibXML::Reader->new(
+ string => "<text>$$text_buffer_ref</text>",
+ huge => 1
+ );
+
+ # Turn reader into XML::CompactTree structure
+ my $tree_data = XML::CompactTree::XS::readSubtreeToPerl($reader, XCT_PARAM);
+
+ # Recursively parse all children
+ $self->_descend(1, $tree_data->[2]);
+};
+
+
+# Recursively called function to handle XML tree data
+sub _descend {
+ my $self = shift;
+
+ # recursion level
+ # (1 = topmost level inside _descend() = should always be level of tag $_TEXT_BODY)
+ my $depth = shift;
+
+ # Iteration through all array elements
+ # ($_[0] is a reference to an array reference)
+ # See notes on how 'XML::CompactTree::XS' works and
+ # see 'NODE TYPES' in manpage of XML::LibXML::Reader
+ foreach my $e (@{$_[0]}) {
+
+ # $e->[1] represents the tag name of an element node
+ # or the primary data of a text or ws node
+ my $node_info = $e->[1];
+
+ # Element node
+ if ($e->[0] == XML_READER_TYPE_ELEMENT) {
+
+ # Deal with opening tag
+
+ # Get the child index depending on the debug state.
+ # This is likely to be optimized away by the compiler.
+ my $children = $e->[DEBUG ? 5 : 4];
+
+ # Skip certain tags
+ if ($self->[SKIP_INLINE_TAGS]->{$node_info}) {
+ $self->_descend($depth + 1, $children) if defined $children;
+ next;
+ };
+
+ my $anno = $self->[STRUCTURES]->add_new_annotation($node_info);
+
+ # Add element also to token list
+ if (!$self->[SKIP_INLINE_TOKENS] && $node_info eq $_TOKENS_TAG) {
+ $self->[TOKENS]->add_annotation($anno);
+ };
+
+ # Handle attributes (if attributes exist)
+ if (defined $e->[3]) {
+
+ # with 'XCT_ATTRIBUTE_ARRAY', $node->[3] is an array reference of the form
+ # [ name1, value1, name2, value2, ....] of attribute names and corresponding values.
+ # NOTE:
+ # arrays are faster (see: http://makepp.sourceforge.net/2.0/perl_performance.html)
+ for (local $_ = 0; $_ < @{$e->[3]}; $_ += 2) {
+ $anno->add_attribute(
+ @{$e->[3]}[$_, $_ + 1]
+ );
+ };
+ };
+
+ my $data = $self->[DATA];
+
+ # This is, where a normal tag or tokens-tag ($_TOKENS_TAG) starts
+ $anno->set_from($data->position + $self->[ADD_ONE]);
+
+ # Call function recursively
+ # do no recursion, if $children is not defined
+ # (because we have no array of child-nodes, e.g.: <back/>)
+ $self->_descend($depth+1, $children) if defined $children;
+
+
+ # Deal with closing tag
+
+ # NOTE:
+ # use $pos, because the offsets are _between_ the characters
+ # (e.g.: word = 'Hello' => from = 0 (before 'H'), to = 5 (after 'o'))
+ my $pos = $data->position;
+
+ # Handle structures and tokens
+
+ my $from = $anno->from;
+
+ my $ws = $self->[WS];
+
+ # ~ whitespace related issue ~
+ if ($from > 0 && not exists $ws->{$from - 1}) {
+
+ # Previous node was a text-node
+ $anno->set_from($from - 1);
+ };
+
+ # in case this fails, check input
+ if (($from - 1) > $pos) {
+ die $log->fatal(
+ 'text_id="' . $self->[TEXT_ID] . '", ' .
+ 'processing of structures: ' .
+ "from-value ($from) is 2 or more greater " .
+ "than to-value ($pos) => please check. Aborting"
+ );
+ };
+
+ # TODO:
+ # find example for which this case applies
+ # maybe this is not necessary anymore, because the
+ # above recorrection of the from-value suffices
+ #
+ # TODO:
+ # check, if it's better to remove this line and
+ # change above check to 'if ($from - 1) >= $pos;
+ # do testing with bigger corpus excerpt (wikipedia?)
+ $anno->set_from($pos) if $from == $pos + 1;
+ $anno->set_to($pos);
+ $anno->set_level($depth);
+
+ # Clean up whitespace
+ delete $ws->{$from - 1} if $from > 0 && exists $ws->{$from - 1};
+ }
+
+ # Text node
+ elsif ($e->[0] == XML_READER_TYPE_TEXT) {
+
+ $self->[ADD_ONE] = 1;
+ $self->[DATA]->append($node_info);
+ }
+
+ # Whitespace node
+ # (See notes on whitespace handling - regarding XML_READER_TYPE_SIGNIFICANT_WHITESPACE)
+ elsif ($e->[0] == XML_READER_TYPE_SIGNIFICANT_WHITESPACE) {
+
+ # state, that this from-index belongs to a whitespace-node
+ # ('++' doesn't mean a thing here - maybe it could be used for a consistency check)
+ $self->[WS]->{$self->[DATA]->position}++;
+
+ $self->[ADD_ONE] = 0;
+ $self->[DATA]->append($node_info);
+ }
+
+ # not yet handled type
+ else {
+
+ die $log->fatal('Not yet handled type ($e->[0]=' . $e->[0] . ') ... => Aborting');
+ };
+ };
+
+ 1;
+};
+
+
+# Return data collector
+sub data {
+ $_[0]->[DATA];
+};
+
+
+# Return structures collector
+sub structures {
+ $_[0]->[STRUCTURES];
+};
+
+
+# Return tokens collector
+sub tokens {
+ $_[0]->[TOKENS];
+};
+
+
+1;
+
+
+__END__
+
+# NOTES
+
+## Notes on how 'XML::CompactTree::XS' works
+
+Example: <node a="v"><node1>some <n/> text</node1><node2>more-text</node2></node>
+
+Print out name of 'node2' for the above example:
+
+echo '<node a="v"><node1>some <n/> text</node1><node2>more-text</node2></node>' | perl -e 'use XML::CompactTree::XS; use XML::LibXML::Reader; $reader = XML::LibXML::Reader->new(IO => STDIN); $data = XML::CompactTree::XS::readSubtreeToPerl( $reader, XCT_DOCUMENT_ROOT | XCT_IGNORE_COMMENTS | XCT_LINE_NUMBERS ); print "\x27".$data->[2]->[0]->[5]->[1]->[1]."\x27\n"'
+
+Exploring the structure of $data ( = reference to below array ):
+
+[ 0: XML_READER_TYPE_DOCUMENT,
+ 1: ?
+ 2: [ 0: [ 0: XML_READER_TYPE_ELEMENT <- start recursion with array '$data->[2]' (see descend( \$tree_data->[2] ))
+ 1: 'node'
+ 2: ?
+ 3: HASH (attributes)
+ 4: 1 (line number)
+ 5: [ 0: [ 0: XML_READER_TYPE_ELEMENT
+ 1: 'node1'
+ 2: ?
+ 3: undefined (no attributes)
+ 4: 1 (line number)
+ 5: [ 0: [ 0: XML_READER_TYPE_TEXT
+ 1: 'some '
+ ]
+ 1: [ 0: XML_READER_TYPE_ELEMENT
+ 1: 'n'
+ 2: ?
+ 3: undefined (no attributes)
+ 4: 1 (line number)
+ 5: undefined (no child-nodes)
+ ]
+ 2: [ 0: XML_READER_TYPE_TEXT
+ 1: ' text'
+ ]
+ ]
+ ]
+ 1: [ 0: XML_READER_TYPE_ELEMENT
+ 1: 'node2'
+ 2: ?
+ 3: undefined (not attributes)
+ 4: 1 (line number)
+ 5: [ 0: [ 0: XML_READER_TYPE_TEXT
+ 1: 'more-text'
+ ]
+ ]
+ ]
+ ]
+ ]
+ ]
+]
+
+$data->[0] = 9 (=> type == XML_READER_TYPE_DOCUMENT)
+
+ref($data->[2]) == ARRAY (with 1 element for 'node')
+ref($data->[2]->[0]) == ARRAY (with 6 elements)
+
+$data->[2]->[0]->[0] == 1 (=> type == XML_READER_TYPE_ELEMENT)
+$data->[2]->[0]->[1] == 'node'
+ref($data->[2]->[0]->[3]) == HASH (=> ${$data->[2]->[0]->[3]}{a} == 'v')
+$data->[2]->[0]->[4] == 1 (line number)
+ref($data->[2]->[0]->[5]) == ARRAY (with 2 elements for 'node1' and 'node2')
+ # child-nodes of actual node (see $children)
+
+ref($data->[2]->[0]->[5]->[0]) == ARRAY (with 6 elements)
+$data->[2]->[0]->[5]->[0]->[0] == 1 (=> type == XML_READER_TYPE_ELEMENT)
+$data->[2]->[0]->[5]->[0]->[1] == 'node1'
+$data->[2]->[0]->[5]->[0]->[3] == undefined (=> no attribute)
+$data->[2]->[0]->[5]->[0]->[4] == 1 (line number)
+ref($data->[2]->[0]->[5]->[0]->[5]) == ARRAY (with 3 elements for 'some ', '<n/>' and ' text')
+
+ref($data->[2]->[0]->[5]->[0]->[5]->[0]) == ARRAY (with 2 elements)
+$data->[2]->[0]->[5]->[0]->[5]->[0]->[0] == 3 (=> type == XML_READER_TYPE_TEXT)
+$data->[2]->[0]->[5]->[0]->[5]->[0]->[1] == 'some '
+
+ref($data->[2]->[0]->[5]->[0]->[5]->[1]) == ARRAY (with 5 elements)
+$data->[2]->[0]->[5]->[0]->[5]->[1]->[0] == 1 (=> type == XML_READER_TYPE_ELEMENT)
+$data->[2]->[0]->[5]->[0]->[5]->[1]->[1] == 'n'
+$data->[2]->[0]->[5]->[0]->[5]->[1]->[3] == undefined (=> no attribute)
+$data->[2]->[0]->[5]->[0]->[5]->[1]->[4] == 1 (line number)
+$data->[2]->[0]->[5]->[0]->[5]->[1]->[5] == undefined (=> no child-nodes)
+
+ref($data->[2]->[0]->[5]->[0]->[5]->[2]) == ARRAY (with 2 elements)
+$data->[2]->[0]->[5]->[0]->[5]->[2]->[0] == 3 (=> type == XML_READER_TYPE_TEXT)
+$data->[2]->[0]->[5]->[0]->[5]->[2]->[1] == ' text'
+
+
+descend() starts with the array reference ${$_[0]} (= \$tree_data->[2]), which corresponds to ${\$data->[2]} in the above example.
+Hence, the expression @{${$_[0]}} corresponds to @{${\$data->[2]}}, $e to ${${\$data->[2]}}[0] (= $data->[2]->[0]) and $e->[0] to
+${${\$data->[2]}}[0]->[0] (= $data->[2]->[0]->[0]).
+
+## Notes on whitespace handling
+
+Every whitespace inside the processed text is 'significant' and recognized as a node of type 'XML_READER_TYPE_SIGNIFICANT_WHITESPACE'
+(see function 'descend()').
+
+Definition of significant and insignificant whitespace
+(source: https://www.oracle.com/technical-resources/articles/wang-whitespace.html):
+
+Significant whitespace is part of the document content and should be preserved.
+Insignificant whitespace is used when editing XML documents for readability.
+These whitespaces are typically not intended for inclusion in the delivery of the document.
+
+### Regarding XML_READER_TYPE_SIGNIFICANT_WHITESPACE
+
+The 3rd form of nodes, besides text- (XML_READER_TYPE_TEXT) and tag-nodes (XML_READER_TYPE_ELEMENT) are nodes of the type
+ 'XML_READER_TYPE_SIGNIFICANT_WHITESPACE'.
+
+When modifiying the previous example (see: Notes on how 'XML::CompactTree::XS' works) by inserting an additional blank between
+ '</node1>' and '<node2>', the output for '$data->[2]->[0]->[5]->[1]->[1]' is a blank (' ') and it's type is '14'
+ (XML_READER_TYPE_SIGNIFICANT_WHITESPACE, see 'man XML::LibXML::Reader'):
+
+echo '<node a="v"><node1>some <n/> text</node1> <node2>more-text</node2></node>' | perl -e 'use XML::CompactTree::XS; use XML::LibXML::Reader; $reader = XML::LibXML::Reader->new(IO => STDIN); $data = XML::CompactTree::XS::readSubtreeToPerl( $reader, XCT_DOCUMENT_ROOT | XCT_IGNORE_COMMENTS | XCT_LINE_NUMBERS ); print "node=\x27".$data->[2]->[0]->[5]->[1]->[1]."\x27, type=".$data->[2]->[0]->[5]->[1]->[0]."\n"'
+
+
+Example: '... <head type="main"><s>Campagne in Frankreich</s></head><head type="sub"> <s>1792</s> ...'
+
+Two text-nodes should normally be separated by a blank. In the above example, that would be the 2 text-nodes
+ 'Campagne in Frankreich' and '1792', which are separated by the whitespace-node ' ' (see [2]).
+
+The text-node 'Campagne in Frankreich' leads to the setting of '$add_one' to 1, so that when opening the 2nd 'head'-tag,
+ it's from-index gets set to the correct start-index of '1792' (and not to the start-index of the whitespace-node ' ').
+
+The assumption here is, that in most cases there _is_ a whitespace node between 2 text-nodes. The below code fragment
+ enables a way, to check, if this really _was_ the case for the last 2 'non-tag'-nodes, when closing a tag:
+
+When a whitespace-node is read, its from-index is stored as a hash-key (in %ws), to state that it belongs to a ws-node.
+ So when closing a tag, it can be checked, if the previous 'non-tag'-node (text or whitespace), which is the one before
+ the last read 'non-tag'-node, was a actually _not_ a ws-node, but instead a text-node. In that case, the from-value of
+ the last read 'non-tag'-node has to be corrected (see [1]),
+
+For whitespace-nodes $add_one is set to 0, so when opening the next tag (in the above example the 2nd 's'-tag), no
+ additional 1 is added (because this was already done by the whitespace-node itself when incrementing the variable $pos).
+
+[1]
+Now, what happens, when 2 text-nodes are _not_ seperated by a whitespace-node (e.g.: <w>Augen<c>,</c></w>)?
+ In this case, the falsely increased from-value has to be decreased again by 1 when closing the enclosing tag
+ (see above code fragment '... not exists $ws{ $from - 1 } ...').
+
+[2]
+Comparing the 2 examples '<w>fu</w> <w>bar</w>' and '<w>fu</w><w> </w><w>bar</w>', is ' ' in both cases handled as a
+ whitespace-node (XML_READER_TYPE_SIGNIFICANT_WHITESPACE).
+
+The from-index of the 2nd w-tag in the second example refers to 'bar', which may not have been the intention
+ (even though '<w> </w>' doesn't make a lot of sense). TODO: could this be a bug?
+
+Empty tags also cling to the next text-token - e.g. in '<w>tok1</w> <w>tok2</w><a><b/></a> <w>tok3</w>' are the from-
+ and to-indizes for the tags 'a' and 'b' both 12, which is the start-index of the token 'tok3'.
+
+
+## Notes on whitespace fixing
+
+The idea for the below code fragment was to fix (recreate) missing whitespace in a poorly created corpus, in which linebreaks where inserted
+ into the text with the addition that maybe (or not) whitespace before those linebreaks was unintenionally stripped.
+
+It soon turned out, that it was best to suggest considering just avoiding linebreaks and putting all primary text tokens into one line (see
+ example further down and notes on 'Input restrictions' in the manpage).
+
+Somehow an old first very poor approach remained, which is not stringent, but also doesn't affect one-line text.
+
+Examples (how primary text with linebreaks would be converted by below code):
+
+ '...<w>end</w>\n<w>.</w>...' -> '...<w>end</w> <w>.</w>...'
+ '...<w>,</w>\n<w>this</w>\n<w>is</w>\n<w>it</w>\n<w>!</w>...' -> '<w>,<w> <w>this</w> <w>is</w> <w>it</w> <w>!</w>'.
+
+Blanks are inserted before the 1st character:
+
+ NOTE: not stringent ('...' stands for text):
+
+ beg1............................end1 => no blank before 'beg1'
+ beg2....<pb/>...................end2 => no blank before 'beg2'
+ beg3....<info attr1="val1"/>....end3 => no blank before 'beg3'
+ beg4....<test>ok</test>.........end4 => blank before 'beg4'
+
+ => beg1....end1beg2...<pb/>...end2beg3....<info attr1="val1"/>....end3 beg4...<test>ok</test>....end4
+ ^
+ |_blank between 'end3' and 'beg4'
diff --git a/script/tei2korapxml b/script/tei2korapxml
index 80b0596..5740407 100755
--- a/script/tei2korapxml
+++ b/script/tei2korapxml
@@ -11,9 +11,6 @@
use Encode qw(decode);
-use XML::CompactTree::XS;
-use XML::LibXML::Reader;
-
use FindBin;
BEGIN {
unshift @INC, "$FindBin::Bin/../lib";
@@ -23,10 +20,9 @@
use KorAP::XML::TEI::Tokenizer::External;
use KorAP::XML::TEI::Tokenizer::Conservative;
use KorAP::XML::TEI::Tokenizer::Aggressive;
-use KorAP::XML::TEI::Annotations::Collector;
-use KorAP::XML::TEI::Data;
use KorAP::XML::TEI::Zipper;
use KorAP::XML::TEI::Header;
+use KorAP::XML::TEI::Inline;
eval {
require KorAP::XML::TEI::Tokenizer::KorAP;
@@ -39,16 +35,7 @@
use constant {
# Set to 1 for minimal more debug output (no need to be parametrized)
- DEBUG => $ENV{KORAPXMLTEI_DEBUG} // 0,
-
- # XCT_LINE_NUMBERS is only needed for debugging
- # (see XML::CompactTree::XS)
- XCT_PARAM => (
- XCT_DOCUMENT_ROOT
- | XCT_IGNORE_COMMENTS
- | XCT_ATTRIBUTE_ARRAY
- | ($ENV{KORAPXMLTEI_DEBUG} ? XCT_LINE_NUMBERS : 0)
- )
+ DEBUG => $ENV{KORAPXMLTEI_DEBUG} // 0
};
# Parse options from the command line
@@ -99,9 +86,6 @@
# TODO: IDS-specific (and redundant)
my $_HEADER_TAG = 'idsHeader';
-# name of the tag containing all information stored in $_tokens_file
-my $_TOKENS_TAG = 'w';
-
# Define tokenizers
if ($use_tokenizer_sentence_splits && !$tokenizer_korap) {
@@ -149,45 +133,21 @@
# Handling inline annotations (inside $_TOKENS_TAG)
my $_INLINE_ANNOT = $ENV{KORAPXMLTEI_INLINE} ? 1 : 0;
-# Initialize Token- and Structure-Collector
-my $tokens = KorAP::XML::TEI::Annotations::Collector->new;
-my $structures = KorAP::XML::TEI::Annotations::Collector->new;
-
-# Initialize Data-Collector
-my $data = KorAP::XML::TEI::Data->new;
-
# Initialize zipper
my $zipper = KorAP::XML::TEI::Zipper->new($root_dir);
-
# text directory (below $root_dir)
my $dir = '';
# Escaped version of text id
my $text_id_esc;
-# element from $tree_data
-my $e;
-
# Default encoding of the text
my $input_enc = 'UTF-8';
-# variables for handling ~ whitespace related issue ~
-# (it is sometimes necessary, to correct the from-values for some tags)
-my $add_one;
-
# text line (needed for whitespace handling)
my $text_line = 0;
-# hash for indices of whitespace-nodes
-# (needed to recorrect from-values)
-# IDEA:
-# when closing element, check if it's from-index minus 1 refers to a whitespace-node
-# (means: 'from-index - 1' is a key in %ws).
-# if this is _not_ the case, then the from-value is one
-# to high => correct it by substracting 1
-my %ws;
-
# Input file handle (default: stdin)
my $input_fh = *STDIN;
@@ -202,6 +162,13 @@
binmode $input_fh;
+# Create inline parser object
+my $inline = KorAP::XML::TEI::Inline->new(
+ $skip_inline_tokens,
+ \%skip_inline_tags
+);
+
+
# Reading input document
MAIN: while (<$input_fh>) {
@@ -252,29 +219,20 @@
if ($dir eq '') {
$log->warn(
"Maybe empty textSigle => skipping this text ...\n" .
- 'data=' . substr($data->data, 0, 200)
+ 'data=' . substr($inline->data->data, 0, 200)
);
next MAIN;
};
- my $reader = XML::LibXML::Reader->new(
- string => "<text>$text_buffer</text>",
- huge => 1
- );
-
- my $tree_data = XML::CompactTree::XS::readSubtreeToPerl($reader, XCT_PARAM);
-
- # ~ whitespace related issue ~
- $add_one = 0;
- %ws = ();
-
- # Recursively parse all children
- descend(1, $tree_data->[2]);
+ # Parse inline structure
+ $inline->parse($text_id_esc, \$text_buffer);
if (DEBUG) {
$log->debug("Writing (utf8-formatted) xml file $dir/${data_file}.xml");
};
+ my $data = $inline->data;
+
# Write data.xml
$data->to_zip(
$zipper->new_stream("$dir/${data_file}.xml"),
@@ -291,7 +249,7 @@
);
if ($use_tokenizer_sentence_splits) {
- $ext_tok->sentencize_from_previous_input($structures);
+ $ext_tok->sentencize_from_previous_input($inline->structures);
};
};
@@ -311,29 +269,26 @@
};
# ~ write structures ~
- if (!$structures->empty) {
- $structures->to_zip(
+ if (!$inline->structures->empty) {
+ $inline->structures->to_zip(
$zipper->new_stream("$dir/$_structure_dir/${_structure_file}.xml"),
$text_id_esc,
2 # = structure serialization
- )->reset;
+ );
};
# ~ write tokens ~
- unless ($skip_inline_tokens || $tokens->empty) {
- $tokens->to_zip(
+ unless ($skip_inline_tokens || $inline->tokens->empty) {
+ $inline->tokens->to_zip(
$zipper->new_stream("$dir/$_tokens_dir/${_tokens_file}.xml"),
$text_id_esc,
$_INLINE_ANNOT # Either 0 = tokens without inline or 1 = tokens with inline
- )->reset;
+ );
};
# reinit.
$dir = '';
- # Maybe not necessary
- $data->reset;
-
next MAIN;
};
@@ -427,144 +382,6 @@
close $input_fh;
-exit(0);
-
-
-# Recursively called function to handle XML tree data
-sub descend {
-
- # recursion level
- # (1 = topmost level inside descend() = should always be level of tag $_TEXT_BODY)
- my $depth = shift;
-
- # Iteration through all array elements
- # ($_[0] is a reference to an array reference)
- # See notes on how 'XML::CompactTree::XS' works and
- # see 'NODE TYPES' in manpage of XML::LibXML::Reader
- foreach $e (@{$_[0]}) {
-
- # $e->[1] represents the tag name of an element node
- # or the primary data of a text or ws node
- my $node_info = $e->[1];
-
- # Element node
- if ($e->[0] == XML_READER_TYPE_ELEMENT) {
-
- # Deal with opening tag
-
- # Get the child index depending on the debug state.
- # This is likely to be optimized away by the compiler.
- my $children = $e->[DEBUG ? 5 : 4];
-
- # Skip certain tags
- if ($skip_inline_tags{$node_info}) {
- descend($depth + 1, $children) if defined $children;
- next;
- };
-
- my $anno = $structures->add_new_annotation($node_info);
-
- # Add element also to token list
- if (!$skip_inline_tokens && $node_info eq $_TOKENS_TAG) {
- $tokens->add_annotation($anno);
- };
-
- # Handle attributes (if attributes exist)
- if (defined $e->[3]) {
-
- # with 'XCT_ATTRIBUTE_ARRAY', $node->[3] is an array reference of the form
- # [ name1, value1, name2, value2, ....] of attribute names and corresponding values.
- # NOTE:
- # arrays are faster (see: http://makepp.sourceforge.net/2.0/perl_performance.html)
- for (local $_ = 0; $_ < @{$e->[3]}; $_ += 2) {
- $anno->add_attribute(
- @{$e->[3]}[$_, $_ + 1]
- );
- };
- };
-
- # this is, where a normal tag or tokens-tag ($_TOKENS_TAG) starts
- $anno->set_from($data->position + $add_one);
-
-
- # Call function recursively
- # do no recursion, if $children is not defined
- # (because we have no array of child-nodes, e.g.: <back/>)
- descend($depth+1, $children) if defined $children;
-
-
- # Deal with closing tag
-
- # NOTE:
- # use $pos, because the offsets are _between_ the characters
- # (e.g.: word = 'Hello' => from = 0 (before 'H'), to = 5 (after 'o'))
- my $pos = $data->position;
-
- # Handle structures and tokens
-
- my $from = $anno->from;
-
- # ~ whitespace related issue ~
- if ($from > 0 && not exists $ws{$from - 1}) {
-
- # Previous node was a text-node
- $anno->set_from($from - 1);
- };
-
- # in case this fails, check input
- if (($from - 1) > $pos) {
- die $log->fatal(
- "text_id='$text_id_esc', " .
- 'processing of structures: ' .
- "from-value ($from) is 2 or more greater " .
- "than to-value ($pos) => please check. Aborting"
- );
- };
-
- # TODO:
- # find example for which this case applies
- # maybe this is not necessary anymore, because the
- # above recorrection of the from-value suffices
- #
- # TODO:
- # check, if it's better to remove this line and
- # change above check to 'if ($from - 1) >= $pos;
- # do testing with bigger corpus excerpt (wikipedia?)
- $anno->set_from($pos) if $from == $pos + 1;
- $anno->set_to($pos);
- $anno->set_level($depth);
-
- # Clean up whitespace
- delete $ws{$from - 1} if $from > 0 && exists $ws{$from - 1};
- }
-
- # Text node
- elsif ($e->[0] == XML_READER_TYPE_TEXT) {
-
- $add_one = 1;
- $data->append($node_info);
- }
-
- # Whitespace node
- # (See notes on whitespace handling - regarding XML_READER_TYPE_SIGNIFICANT_WHITESPACE)
- elsif ($e->[0] == XML_READER_TYPE_SIGNIFICANT_WHITESPACE) {
-
- # state, that this from-index belongs to a whitespace-node
- # ('++' doesn't mean a thing here - maybe it could be used for a consistency check)
- $ws{$data->position}++;
-
- $add_one = 0;
- $data->append($node_info);
- }
-
- # not yet handled type
- else {
-
- die $log->fatal('Not yet handled type ($e->[0]=' . $e->[0] . ') ... => Aborting');
- };
- };
-};
-
__END__
@@ -780,186 +597,6 @@
# NOTES
-## Notes on how 'XML::CompactTree::XS' works
-
-Example: <node a="v"><node1>some <n/> text</node1><node2>more-text</node2></node>
-
-Print out name of 'node2' for the above example:
-
-echo '<node a="v"><node1>some <n/> text</node1><node2>more-text</node2></node>' | perl -e 'use XML::CompactTree::XS; use XML::LibXML::Reader; $reader = XML::LibXML::Reader->new(IO => STDIN); $data = XML::CompactTree::XS::readSubtreeToPerl( $reader, XCT_DOCUMENT_ROOT | XCT_IGNORE_COMMENTS | XCT_LINE_NUMBERS ); print "\x27".$data->[2]->[0]->[5]->[1]->[1]."\x27\n"'
-
-Exploring the structure of $data ( = reference to below array ):
-
-[ 0: XML_READER_TYPE_DOCUMENT,
- 1: ?
- 2: [ 0: [ 0: XML_READER_TYPE_ELEMENT <- start recursion with array '$data->[2]' (see descend( \$tree_data->[2] ))
- 1: 'node'
- 2: ?
- 3: HASH (attributes)
- 4: 1 (line number)
- 5: [ 0: [ 0: XML_READER_TYPE_ELEMENT
- 1: 'node1'
- 2: ?
- 3: undefined (no attributes)
- 4: 1 (line number)
- 5: [ 0: [ 0: XML_READER_TYPE_TEXT
- 1: 'some '
- ]
- 1: [ 0: XML_READER_TYPE_ELEMENT
- 1: 'n'
- 2: ?
- 3: undefined (no attributes)
- 4: 1 (line number)
- 5: undefined (no child-nodes)
- ]
- 2: [ 0: XML_READER_TYPE_TEXT
- 1: ' text'
- ]
- ]
- ]
- 1: [ 0: XML_READER_TYPE_ELEMENT
- 1: 'node2'
- 2: ?
- 3: undefined (not attributes)
- 4: 1 (line number)
- 5: [ 0: [ 0: XML_READER_TYPE_TEXT
- 1: 'more-text'
- ]
- ]
- ]
- ]
- ]
- ]
-]
-
-$data->[0] = 9 (=> type == XML_READER_TYPE_DOCUMENT)
-
-ref($data->[2]) == ARRAY (with 1 element for 'node')
-ref($data->[2]->[0]) == ARRAY (with 6 elements)
-
-$data->[2]->[0]->[0] == 1 (=> type == XML_READER_TYPE_ELEMENT)
-$data->[2]->[0]->[1] == 'node'
-ref($data->[2]->[0]->[3]) == HASH (=> ${$data->[2]->[0]->[3]}{a} == 'v')
-$data->[2]->[0]->[4] == 1 (line number)
-ref($data->[2]->[0]->[5]) == ARRAY (with 2 elements for 'node1' and 'node2')
- # child-nodes of actual node (see $children)
-
-ref($data->[2]->[0]->[5]->[0]) == ARRAY (with 6 elements)
-$data->[2]->[0]->[5]->[0]->[0] == 1 (=> type == XML_READER_TYPE_ELEMENT)
-$data->[2]->[0]->[5]->[0]->[1] == 'node1'
-$data->[2]->[0]->[5]->[0]->[3] == undefined (=> no attribute)
-$data->[2]->[0]->[5]->[0]->[4] == 1 (line number)
-ref($data->[2]->[0]->[5]->[0]->[5]) == ARRAY (with 3 elements for 'some ', '<n/>' and ' text')
-
-ref($data->[2]->[0]->[5]->[0]->[5]->[0]) == ARRAY (with 2 elements)
-$data->[2]->[0]->[5]->[0]->[5]->[0]->[0] == 3 (=> type == XML_READER_TYPE_TEXT)
-$data->[2]->[0]->[5]->[0]->[5]->[0]->[1] == 'some '
-
-ref($data->[2]->[0]->[5]->[0]->[5]->[1]) == ARRAY (with 5 elements)
-$data->[2]->[0]->[5]->[0]->[5]->[1]->[0] == 1 (=> type == XML_READER_TYPE_ELEMENT)
-$data->[2]->[0]->[5]->[0]->[5]->[1]->[1] == 'n'
-$data->[2]->[0]->[5]->[0]->[5]->[1]->[3] == undefined (=> no attribute)
-$data->[2]->[0]->[5]->[0]->[5]->[1]->[4] == 1 (line number)
-$data->[2]->[0]->[5]->[0]->[5]->[1]->[5] == undefined (=> no child-nodes)
-
-ref($data->[2]->[0]->[5]->[0]->[5]->[2]) == ARRAY (with 2 elements)
-$data->[2]->[0]->[5]->[0]->[5]->[2]->[0] == 3 (=> type == XML_READER_TYPE_TEXT)
-$data->[2]->[0]->[5]->[0]->[5]->[2]->[1] == ' text'
-
-
-descend() starts with the array reference ${$_[0]} (= \$tree_data->[2]), which corresponds to ${\$data->[2]} in the above example.
-Hence, the expression @{${$_[0]}} corresponds to @{${\$data->[2]}}, $e to ${${\$data->[2]}}[0] (= $data->[2]->[0]) and $e->[0] to
-${${\$data->[2]}}[0]->[0] (= $data->[2]->[0]->[0]).
-
-
-## Notes on whitespace handling
-
-Every whitespace inside the processed text is 'significant' and recognized as a node of type 'XML_READER_TYPE_SIGNIFICANT_WHITESPACE'
-(see function 'descend()').
-
-Definition of significant and insignificant whitespace
-(source: https://www.oracle.com/technical-resources/articles/wang-whitespace.html):
-
-Significant whitespace is part of the document content and should be preserved.
-Insignificant whitespace is used when editing XML documents for readability.
-These whitespaces are typically not intended for inclusion in the delivery of the document.
-
-### Regarding XML_READER_TYPE_SIGNIFICANT_WHITESPACE
-
-The 3rd form of nodes, besides text- (XML_READER_TYPE_TEXT) and tag-nodes (XML_READER_TYPE_ELEMENT) are nodes of the type
- 'XML_READER_TYPE_SIGNIFICANT_WHITESPACE'.
-
-When modifiying the previous example (see: Notes on how 'XML::CompactTree::XS' works) by inserting an additional blank between
- '</node1>' and '<node2>', the output for '$data->[2]->[0]->[5]->[1]->[1]' is a blank (' ') and it's type is '14'
- (XML_READER_TYPE_SIGNIFICANT_WHITESPACE, see 'man XML::LibXML::Reader'):
-
-echo '<node a="v"><node1>some <n/> text</node1> <node2>more-text</node2></node>' | perl -e 'use XML::CompactTree::XS; use XML::LibXML::Reader; $reader = XML::LibXML::Reader->new(IO => STDIN); $data = XML::CompactTree::XS::readSubtreeToPerl( $reader, XCT_DOCUMENT_ROOT | XCT_IGNORE_COMMENTS | XCT_LINE_NUMBERS ); print "node=\x27".$data->[2]->[0]->[5]->[1]->[1]."\x27, type=".$data->[2]->[0]->[5]->[1]->[0]."\n"'
-
-
-Example: '... <head type="main"><s>Campagne in Frankreich</s></head><head type="sub"> <s>1792</s> ...'
-
-Two text-nodes should normally be separated by a blank. In the above example, that would be the 2 text-nodes
- 'Campagne in Frankreich' and '1792', which are separated by the whitespace-node ' ' (see [2]).
-
-The text-node 'Campagne in Frankreich' leads to the setting of '$add_one' to 1, so that when opening the 2nd 'head'-tag,
- it's from-index gets set to the correct start-index of '1792' (and not to the start-index of the whitespace-node ' ').
-
-The assumption here is, that in most cases there _is_ a whitespace node between 2 text-nodes. The below code fragment
- enables a way, to check, if this really _was_ the case for the last 2 'non-tag'-nodes, when closing a tag:
-
-When a whitespace-node is read, its from-index is stored as a hash-key (in %ws), to state that it belongs to a ws-node.
- So when closing a tag, it can be checked, if the previous 'non-tag'-node (text or whitespace), which is the one before
- the last read 'non-tag'-node, was a actually _not_ a ws-node, but instead a text-node. In that case, the from-value of
- the last read 'non-tag'-node has to be corrected (see [1]),
-
-For whitespace-nodes $add_one is set to 0, so when opening the next tag (in the above example the 2nd 's'-tag), no
- additional 1 is added (because this was already done by the whitespace-node itself when incrementing the variable $pos).
-
-[1]
-Now, what happens, when 2 text-nodes are _not_ seperated by a whitespace-node (e.g.: <w>Augen<c>,</c></w>)?
- In this case, the falsely increased from-value has to be decreased again by 1 when closing the enclosing tag
- (see above code fragment '... not exists $ws{ $from - 1 } ...').
-
-[2]
-Comparing the 2 examples '<w>fu</w> <w>bar</w>' and '<w>fu</w><w> </w><w>bar</w>', is ' ' in both cases handled as a
- whitespace-node (XML_READER_TYPE_SIGNIFICANT_WHITESPACE).
-
-The from-index of the 2nd w-tag in the second example refers to 'bar', which may not have been the intention
- (even though '<w> </w>' doesn't make a lot of sense). TODO: could this be a bug?
-
-Empty tags also cling to the next text-token - e.g. in '<w>tok1</w> <w>tok2</w><a><b/></a> <w>tok3</w>' are the from-
- and to-indizes for the tags 'a' and 'b' both 12, which is the start-index of the token 'tok3'.
-
-
-## Notes on whitespace fixing
-
-The idea for the below code fragment was to fix (recreate) missing whitespace in a poorly created corpus, in which linebreaks where inserted
- into the text with the addition that maybe (or not) whitespace before those linebreaks was unintenionally stripped.
-
-It soon turned out, that it was best to suggest considering just avoiding linebreaks and putting all primary text tokens into one line (see
- example further down and notes on 'Input restrictions' in the manpage).
-
-Somehow an old first very poor approach remained, which is not stringent, but also doesn't affect one-line text.
-
-Examples (how primary text with linebreaks would be converted by below code):
-
- '...<w>end</w>\n<w>.</w>...' -> '...<w>end</w> <w>.</w>...'
- '...<w>,</w>\n<w>this</w>\n<w>is</w>\n<w>it</w>\n<w>!</w>...' -> '<w>,<w> <w>this</w> <w>is</w> <w>it</w> <w>!</w>'.
-
-Blanks are inserted before the 1st character:
-
- NOTE: not stringent ('...' stands for text):
-
- beg1............................end1 => no blank before 'beg1'
- beg2....<pb/>...................end2 => no blank before 'beg2'
- beg3....<info attr1="val1"/>....end3 => no blank before 'beg3'
- beg4....<test>ok</test>.........end4 => blank before 'beg4'
-
- => beg1....end1beg2...<pb/>...end2beg3....<info attr1="val1"/>....end3 beg4...<test>ok</test>....end4
- ^
- |_blank between 'end3' and 'beg4'
-
-
## Notes on segfault prevention
binmode on the input handler prevents segfaulting of 'XML::LibXML::Reader' inside the main loop
diff --git a/t/inline.t b/t/inline.t
new file mode 100644
index 0000000..7fa2357
--- /dev/null
+++ b/t/inline.t
@@ -0,0 +1,97 @@
+use strict;
+use warnings;
+
+use FindBin;
+BEGIN {
+ unshift @INC, "$FindBin::Bin/../lib";
+};
+
+use Test::More;
+use Test::XML::Loy;
+use_ok('KorAP::XML::TEI::Inline');
+
+
+my $inline = KorAP::XML::TEI::Inline->new;
+
+ok($inline->parse('aaa', \'Der <b>alte</b> Mann'), 'Parsed');
+
+is($inline->data->data, 'Der alte Mann');
+
+Test::XML::Loy->new($inline->structures->to_string('aaa', 2))
+ ->attr_is('#s0', 'l', "1")
+ ->attr_is('#s0', 'to', 13)
+ ->text_is('#s0 fs f[name=name]', 'text')
+ ->attr_is('#s1', 'l', "2")
+ ->attr_is('#s1', 'from', 4)
+ ->attr_is('#s1', 'to', 8)
+ ->text_is('#s1 fs f[name=name]', 'b')
+ ;
+
+Test::XML::Loy->new($inline->tokens->to_string('aaa', 0))
+ ->element_exists_not('fs')
+ ;
+
+
+ok($inline->parse('aaa', \'<w>Die</w> <w>alte</w> <w>Frau</w>'), 'Parsed');
+
+is($inline->data->data, 'Die alte Frau');
+
+Test::XML::Loy->new($inline->structures->to_string('aaa', 2))
+ ->attr_is('#s0', 'l', "1")
+ ->attr_is('#s0', 'to', 13)
+ ->text_is('#s0 fs f[name=name]', 'text')
+
+ ->attr_is('#s1', 'l', "2")
+ ->attr_is('#s1', 'to', 3)
+ ->text_is('#s1 fs f[name=name]', 'w')
+
+ ->attr_is('#s2', 'l', "2")
+ ->attr_is('#s2', 'from', 4)
+ ->attr_is('#s2', 'to', 8)
+ ->text_is('#s2 fs f[name=name]', 'w')
+
+ ->attr_is('#s3', 'l', "2")
+ ->attr_is('#s3', 'from', 9)
+ ->attr_is('#s3', 'to', 13)
+ ->text_is('#s3 fs f[name=name]', 'w')
+ ;
+
+Test::XML::Loy->new($inline->tokens->to_string('aaa', 0))
+ ->attr_is('#s0', 'l', "2")
+ ->attr_is('#s0', 'to', 3)
+
+ ->attr_is('#s1', 'l', "2")
+ ->attr_is('#s1', 'from', 4)
+ ->attr_is('#s1', 'to', 8)
+
+ ->attr_is('#s2', 'l', "2")
+ ->attr_is('#s2', 'from', 9)
+ ->attr_is('#s2', 'to', 13)
+ ;
+
+ok($inline->parse('aaa', \'<w lemma="die" type="det">Die</w> <w
+ lemma="alt" type="ADJ">alte</w> <w lemma="frau" type="NN">Frau</w>'), 'Parsed');
+
+is($inline->data->data, 'Die alte Frau');
+
+Test::XML::Loy->new($inline->tokens->to_string('aaa', 1))
+ ->attr_is('#s0', 'l', "2")
+ ->attr_is('#s0', 'to', 3)
+ ->text_is('#s0 fs f[name="lemma"]', 'die')
+ ->text_is('#s0 fs f[name="type"]', 'det')
+
+ ->attr_is('#s1', 'l', "2")
+ ->attr_is('#s1', 'from', 4)
+ ->attr_is('#s1', 'to', 8)
+ ->text_is('#s1 fs f[name="lemma"]', 'alt')
+ ->text_is('#s1 fs f[name="type"]', 'ADJ')
+
+ ->attr_is('#s2', 'l', "2")
+ ->attr_is('#s2', 'from', 9)
+ ->attr_is('#s2', 'to', 13)
+ ->text_is('#s2 fs f[name="lemma"]', 'frau')
+ ->text_is('#s2 fs f[name="type"]', 'NN')
+ ;
+
+
+done_testing;