Merge "Add tool to rewrite licenses for the sample corpus"
diff --git a/Changes b/Changes
index 267bf0c..9acb4a1 100644
--- a/Changes
+++ b/Changes
@@ -1,3 +1,13 @@
+0.40 2020-03-03
+ - Fixed XIP parser.
+ - Added example corpus of the
+ Redewiedergabe-Korpus.
+ - Fixed span offset bug.
+ - Fixed milestones behind the last
+ token bug.
+ - Fixed gap behind last token bug.
+ - Fixed <base/s:t> length.
+
0.39 2020-02-19
- Added Talismane support.
- Added "distributor" field to I5 metadata.
diff --git a/lib/KorAP/XML/Annotation/DGD/Structure.pm b/lib/KorAP/XML/Annotation/DGD/Structure.pm
index 6961b45..347b6c0 100644
--- a/lib/KorAP/XML/Annotation/DGD/Structure.pm
+++ b/lib/KorAP/XML/Annotation/DGD/Structure.pm
@@ -73,7 +73,7 @@
o_start => $o_start,
o_end => $_->[1],
p_start => $p_start,
- p_end => $_->[0] + 1,
+ p_end => $_->[0],
pti => 64
);
$mt->payload('<b>1');
diff --git a/lib/KorAP/XML/Annotation/DeReKo/Structure.pm b/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
index 769e629..cc2b751 100644
--- a/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
+++ b/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
@@ -36,6 +36,16 @@
# Get the mtt
my $mtt = $stream->pos($p_start);
+ unless ($mtt) {
+
+ # This is a special case were a milestone is at the
+ # end of a text and can't be indexed at the moment!
+ $$self->log->warn(
+ 'Span ' . $span->to_string . ' can\'t be indexed'
+ );
+ return;
+ };
+
my $p_end = $span->p_end;
# Add structure
diff --git a/lib/KorAP/XML/Annotation/Mate/Dependency.pm b/lib/KorAP/XML/Annotation/Mate/Dependency.pm
index ffc3223..6d242c9 100644
--- a/lib/KorAP/XML/Annotation/Mate/Dependency.pm
+++ b/lib/KorAP/XML/Annotation/Mate/Dependency.pm
@@ -30,126 +30,126 @@
# Iterate over relations
foreach (@$rel) {
- my $label = $_->{-label};
+ my $label = $_->{-label};
- # Relation type is unary
- # Unary means, it refers to itself!
- if ($_->{-type} && $_->{-type} eq 'unary') {
+ # Relation type is unary
+ # Unary means, it refers to itself!
+ if ($_->{-type} && $_->{-type} eq 'unary') {
- # I have no clue, what -- should mean
- # next if $_->{-label} eq '--';
+ # I have no clue, what -- should mean
+ # next if $_->{-label} eq '--';
- # Get target node - not very elegant
- # This is only necessary for nodes with attributes
- # my $target = $stream->get_node(
- # $source, 'mate/d:' . $NODE_LABEL
- # );
+ # Get target node - not very elegant
+ # This is only necessary for nodes with attributes
+ # my $target = $stream->get_node(
+ # $source, 'mate/d:' . $NODE_LABEL
+ # );
- # Target is at the same position!
- my $pos = $source->pos;
+ # Target is at the same position!
+ my $pos = $source->pos;
- my %rel = (
- pti => 32, # term-to-term relation
- payload =>
- '<i>' . $pos # . # right part token position
-# '<s>0' . # $target->tui . # left part tui
-# '<s>0' # . $target->tui # right part tui
- );
+ my %rel = (
+ pti => 32, # term-to-term relation
+ payload =>
+ '<i>' . $pos # . # right part token position
+ # '<s>0' . # $target->tui . # left part tui
+ # '<s>0' # . $target->tui # right part tui
+ );
- # Add relations
- $mtt->add(
- term => '>:mate/d:' . $label,
- %rel
- );
- $mtt->add(
- term => '<:mate/d:' . $label,
- %rel
- );
- }
+ # Add relations
+ $mtt->add(
+ term => '>:mate/d:' . $label,
+ %rel
+ );
+ $mtt->add(
+ term => '<:mate/d:' . $label,
+ %rel
+ );
+ }
- # Not unary
- elsif (!$_->{type}) {
+ # Not unary
+ elsif (!$_->{type}) {
- # Get information about the target token
- my $from = $_->{span}->{-from};
- my $to = $_->{span}->{-to};
+ # Get information about the target token
+ my $from = $_->{span}->{-from};
+ my $to = $_->{span}->{-to};
- # Get source node
- # This is only necessary for nodes with attributes
- # my $source_term = $stream->get_node(
- # $source, 'mate/d:' . $NODE_LABEL
- # );
+ # Get source node
+ # This is only necessary for nodes with attributes
+ # my $source_term = $stream->get_node(
+ # $source, 'mate/d:' . $NODE_LABEL
+ # );
- # Target
- my $target = $tokens->token($from, $to);
+ # Target
+ my $target = $tokens->token($from, $to);
- # Relation is term-to-term with a found target!
- if ($target) {
+ # Relation is term-to-term with a found target!
+ if ($target) {
- # Get target node
- # my $target_term = $stream->get_node(
- # $target, 'mate/d:' . $NODE_LABEL
- # );
+ # Get target node
+ # my $target_term = $stream->get_node(
+ # $target, 'mate/d:' . $NODE_LABEL
+ # );
- $mtt->add(
- term => '>:mate/d:' . $label,
- pti => 32, # term-to-term relation
- payload =>
- '<i>' . $target->pos # . # right part token position
-# '<s>0' . # $source_term->tui . # left part tui
-# '<s>0' # . $target_term->tui # right part tui
- );
+ $mtt->add(
+ term => '>:mate/d:' . $label,
+ pti => 32, # term-to-term relation
+ payload =>
+ '<i>' . $target->pos # . # right part token position
+ # '<s>0' . # $source_term->tui . # left part tui
+ # '<s>0' # . $target_term->tui # right part tui
+ );
- my $target_mtt = $stream->pos($target->pos);
- $target_mtt->add(
- term => '<:mate/d:' . $label,
- pti => 32, # term-to-term relation
- payload =>
- '<i>' . $source->pos # . # left part token position
-# '<s>0' . # $source_term->tui . # left part tui
-# '<s>0' # . $target_term->tui # right part tui
- );
- }
+ my $target_mtt = $stream->pos($target->pos);
+ $target_mtt->add(
+ term => '<:mate/d:' . $label,
+ pti => 32, # term-to-term relation
+ payload =>
+ '<i>' . $source->pos # . # left part token position
+ # '<s>0' . # $source_term->tui . # left part tui
+ # '<s>0' # . $target_term->tui # right part tui
+ );
+ }
- # Relation is possibly term-to-element with a found target!
- elsif ($target = $tokens->span($from, $to)) {
+ # Relation is possibly term-to-element with a found target!
+ elsif ($target = $tokens->span($from, $to)) {
- # Get target node
- # my $target_span = $stream->get_node(
- # $target, 'mate/d:' . $NODE_LABEL
- # );
+ # Get target node
+ # my $target_span = $stream->get_node(
+ # $target, 'mate/d:' . $NODE_LABEL
+ # );
- $mtt->add(
- term => '>:mate/d:' . $label,
- pti => 33, # term-to-element relation
- payload =>
- '<i>' . $target->o_start . # end position
- '<i>' . $target->o_end . # end position
- '<i>' . $target->p_start . # right part start position
- '<i>' . $target->p_end # . # right part end position
-# '<s>0' . # $source_term->tui . # left part tui
-# '<s>0' # . $target_span->tui # right part tui
- );
+ $mtt->add(
+ term => '>:mate/d:' . $label,
+ pti => 33, # term-to-element relation
+ payload =>
+ '<i>' . $target->o_start . # end position
+ '<i>' . $target->o_end . # end position
+ '<i>' . $target->p_start . # right part start position
+ '<i>' . $target->p_end # . # right part end position
+ # '<s>0' . # $source_term->tui . # left part tui
+ # '<s>0' # . $target_span->tui # right part tui
+ );
- my $target_mtt = $stream->pos($target->p_start);
- $target_mtt->add(
- term => '<:mate/d:' . $label,
- pti => 34, # element-to-term relation
- payload =>
- '<i>' . $target->o_start . # end position
- '<i>' . $target->o_end . # end position
- '<i>' . $target->p_end . # right part end position
- '<i>' . $source->pos # . # left part token position
-# '<s>0' . # $source_term->tui . # left part tui
-# '<s>0' # . $target_span->tui # right part tui
+ my $target_mtt = $stream->pos($target->p_start);
+ $target_mtt->add(
+ term => '<:mate/d:' . $label,
+ pti => 34, # element-to-term relation
+ payload =>
+ '<i>' . $target->o_start . # end position
+ '<i>' . $target->o_end . # end position
+ '<i>' . $target->p_end . # right part end position
+ '<i>' . $source->pos # . # left part token position
+ # '<s>0' . # $source_term->tui . # left part tui
+ # '<s>0' # . $target_span->tui # right part tui
- );
- }
- else {
- # use Data::Dumper;
- # warn '2###### ' . Dumper($content);
- };
- };
+ );
+ }
+ else {
+ # use Data::Dumper;
+ # warn '2###### ' . Dumper($content);
+ };
+ };
};
}) or return;
diff --git a/lib/KorAP/XML/Annotation/XIP/Constituency.pm b/lib/KorAP/XML/Annotation/XIP/Constituency.pm
index b823dea..7fd05f9 100644
--- a/lib/KorAP/XML/Annotation/XIP/Constituency.pm
+++ b/lib/KorAP/XML/Annotation/XIP/Constituency.pm
@@ -9,7 +9,7 @@
my $self = shift;
# Collect all spans
- my %xip_const;
+ my %xip_const = ();
# Collect all roots
my $xip_const_root = Set::Scalar->new;
@@ -27,29 +27,34 @@
# Collect the span
$xip_const{$span->id} = $span;
+ # warn 'Remember ' . $span->id;
# It's probably a root
$xip_const_root->insert($span->id);
my $rel = $span->hash->{rel} or return;
+
$rel = [$rel] unless ref $rel eq 'ARRAY';
# Iterate over all relations
foreach (@$rel) {
- if ($_->{-label} eq 'dominates') {
+ next if $_->{-label} ne 'dominates';
- my $target = $_->{-target};
- if (!$target && $_->{-uri} &&
- $_->{-uri} =~ $URI_RE) {
- $target = $1;
- };
+ my $target = $_->{-target};
+ if (!$target && $_->{-uri} &&
+ $_->{-uri} =~ $URI_RE) {
+ $target = $1;
+ };
- # The target may not be addressable
- next unless $target;
+ # The target may not be addressable
+ next unless $target;
- # It's definately not a root
- $xip_const_noroot->insert($target);
- };
+ # It's definately not a root
+ $xip_const_noroot->insert($target);
+
+ # if ($target =~ /^s2_n(?:36|58|59|60|40)$/) {
+ # warn 'Probably not a root ' . $target . ' but ' . $span->id;
+ # };
};
}
) or return;
@@ -109,20 +114,29 @@
$rel = [$rel] unless ref $rel eq 'ARRAY';
+ # Iterate over all relations (again ...)
foreach (@$rel) {
next if $_->{-label} ne 'dominates';
- my $target;
- $target = $_->{-target};
- if (!$target && $_->{-uri} && $_->{-uri} =~ $URI_RE) {
- $target = $1;
+ my $target = $_->{-target};
+ if (!$target && $_->{-uri} &&
+ $_->{-uri} =~ $URI_RE) {
+ $target = $1;
};
+ # if ($span->id =~ /^s2_n(?:36|58|59|60|40)$/ && $target =~ /^s2_n(?:36|58|59|60|40)$/) {
+ # warn 'B: ' . $span->id . ' points to ' . $target;
+ # };
+
next unless $target;
my $subspan = delete $xip_const{$target};
- return unless $subspan;
- # warn "Span " . $target . " not found";
+ # warn "A-Forgot about $target: " . ($subspan ? 'yes' : 'no');
+
+ unless ($subspan) {
+ next;
+ };
+ # warn "Span " . $target . " not found";
$this->($subspan, $level + 1);
};
@@ -133,8 +147,13 @@
# Start tree traversal from the root
foreach ($roots->members) {
+ my $obj = delete $xip_const{$_};
- my $obj = delete $xip_const{$_} or next;
+ # warn "B-Forgot about $_: " . ($obj ? 'yes' : 'no');
+
+ unless ($obj) {
+ next;
+ };
$add_const->($obj, 0);
};
@@ -148,4 +167,5 @@
['xip/c=spans']
};
+
1;
diff --git a/lib/KorAP/XML/Krill.pm b/lib/KorAP/XML/Krill.pm
index ec30176..7ad65c5 100644
--- a/lib/KorAP/XML/Krill.pm
+++ b/lib/KorAP/XML/Krill.pm
@@ -16,7 +16,7 @@
use Data::Dumper;
use File::Spec::Functions qw/catdir catfile catpath splitdir splitpath rel2abs/;
-our $VERSION = '0.39';
+our $VERSION = '0.40';
has 'path';
has [qw/text_sigle doc_sigle corpus_sigle/];
diff --git a/lib/KorAP/XML/Tokenizer.pm b/lib/KorAP/XML/Tokenizer.pm
index f9595dd..59ae17f 100644
--- a/lib/KorAP/XML/Tokenizer.pm
+++ b/lib/KorAP/XML/Tokenizer.pm
@@ -138,18 +138,19 @@
$should++;
# Ignore non-word, non-number, and non-verbal tokens per default
+ # '9646' equals the musical pause, used in speech corpora
if ($self->non_verbal_tokens && ord($token) == 9646) {
# Non-verbal token
} elsif (!$self->non_word_tokens && $token !~ /[\w\d]/) {
# TODO: Recognize punctuations!
- # if ($mtt) {
- # my $term = [$token, $from, $to];
- # $mtt->add(
- # term => '.>:'.$token,
- # payload => '<i>'.$from . '<i>' . $to . '<b>' . $distance++
- # );
- # push(@non_word_tokens, $term);
- # }
+ # if ($mtt) {
+ # my $term = [$token, $from, $to];
+ # $mtt->add(
+ # term => '.>:'.$token,
+ # payload => '<i>'.$from . '<i>' . $to . '<b>' . $distance++
+ # );
+ # push(@non_word_tokens, $term);
+ # }
next;
};
@@ -157,12 +158,12 @@
$mtt = $mtts->add;
# while (scalar @non_word_tokens) {
- # local $_ = shift @non_word_tokens;
- # $mtt->add(
- # term => '.<:' . $_->[0],
- # payload => '<i>' . $_->[1] . '<i>' . $_->[2] . '<b>' . --$distance
- # );
- # $distance = 0;
+ # local $_ = shift @non_word_tokens;
+ # $mtt->add(
+ # term => '.<:' . $_->[0],
+ # payload => '<i>' . $_->[1] . '<i>' . $_->[2] . '<b>' . --$distance
+ # );
+ # $distance = 0;
# };
# Add gap for later finding matching positions before or after
@@ -208,15 +209,15 @@
$mtts->pos(0)->add(
term => '<>:base/s:t',
o_start => 0,
- p_end => ($have - 1),
+ p_end => $have,
o_end => $doc->primary->data_length,
payload => '<b>0',
pti => 64
);
- # Create a gap for the
+ # Create a gap for the end
if ($doc->primary->data_length >= ($old - 1)) {
- $range->gap($old, $doc->primary->data_length + 1, $have-1)
+ $range->gap($old, $doc->primary->data_length + 1, $have)
};
# Add info
@@ -256,27 +257,27 @@
my $from = $-[1];
my $to = $+[1];
$mtt->add(
- term => 'i^1:' . substr($os, $from, $from + $to),
- o_start => $from + $o_start,
- o_end => $to + $o_start
+ term => 'i^1:' . substr($os, $from, $from + $to),
+ o_start => $from + $o_start,
+ o_end => $to + $o_start
) unless $to - $from == $l;
};
while ($s =~ /(0+)[^0]/g) {
my $from = $-[1];
my $to = $+[1];
$mtt->add(
- term => 'i^2:' . substr($os, $from, $from + $to),
- o_start => $from + $o_start,
- o_end => $to + $o_start
+ term => 'i^2:' . substr($os, $from, $from + $to),
+ o_start => $from + $o_start,
+ o_end => $to + $o_start
) unless $to - $from == $l;
};
while ($s =~ /(#)/g) {
my $from = $-[1];
my $to = $+[1];
$mtt->add(
- term => 'i^3:' . substr($os, $from, $from + $to),
- o_start => $from + $o_start,
- o_end => $to + $o_start
+ term => 'i^3:' . substr($os, $from, $from + $to),
+ o_start => $from + $o_start,
+ o_end => $to + $o_start
) unless $to - $from == $l;
};
};
@@ -772,10 +773,10 @@
my ($stream, $span) = @_;
my $mtt = $stream->pos($span->p_start);
$mtt->add(
- term => '<>:s',
- o_start => $span->o_start,
- o_end => $span->o_end,
- p_end => $span->p_end
+ term => '<>:s',
+ o_start => $span->o_start,
+ o_end => $span->o_end,
+ p_end => $span->p_end
);
}
);
@@ -804,9 +805,9 @@
# syntax
if ((my $found = $content->at('f[name="pos"]')) && ($found = $found->text)) {
- $mtt->add(
- term => 'cnx_syn:' . $found
- );
+ $mtt->add(
+ term => 'cnx_syn:' . $found
+ );
};
});
diff --git a/lib/KorAP/XML/Tokenizer/Range.pm b/lib/KorAP/XML/Tokenizer/Range.pm
index 366585c..8b3e992 100644
--- a/lib/KorAP/XML/Tokenizer/Range.pm
+++ b/lib/KorAP/XML/Tokenizer/Range.pm
@@ -80,18 +80,21 @@
return $2;
}
else {
- # I am not sure about that ...
- $found;
+ # The current position is likely wrong. The most common non-gap
+ # and non-exact-match is in the situation of
+ # "y<e> z</e>", where e is in the range of the y token.
+ $found + 1;
};
};
sub to_string {
my $self = shift;
- return join('',
- map {'['.join(',',@$_).']'}
- @{$$self->get_range(0,100,'...')}
- ) . '...';
+ return join(
+ '',
+ map {'['.join(',',@$_).']'}
+ @{$$self->get_range(0,100,'...')}
+ ) . '...';
};
1;
diff --git a/lib/KorAP/XML/Tokenizer/Span.pm b/lib/KorAP/XML/Tokenizer/Span.pm
index b359e14..7d7bd55 100644
--- a/lib/KorAP/XML/Tokenizer/Span.pm
+++ b/lib/KorAP/XML/Tokenizer/Span.pm
@@ -2,6 +2,7 @@
use strict;
use warnings;
use Mojo::DOM;
+use Clone;
sub new {
bless [], shift;
@@ -109,4 +110,11 @@
};
};
+
+# Clone the span
+sub clone {
+ return Clone::clone(shift);
+};
+
+
1;
diff --git a/lib/KorAP/XML/Tokenizer/Units.pm b/lib/KorAP/XML/Tokenizer/Units.pm
index 6b75e42..3ace9eb 100644
--- a/lib/KorAP/XML/Tokenizer/Units.pm
+++ b/lib/KorAP/XML/Tokenizer/Units.pm
@@ -8,6 +8,8 @@
has 'have' => 0;
has 'encoding' => 'utf-8';
+use constant DEBUG => 0;
+
sub span {
my $self = shift;
my ($from, $to, $s) = @_;
@@ -19,11 +21,15 @@
$from //= 0;
# The span is invalid
- return unless $from <= $to;
+ unless ($from <= $to) {
+ if (DEBUG) {
+ warn $s->{-id} . ' is invalid';
+ };
+ return;
+ };
my $span = KorAP::XML::Tokenizer::Span->new;
-
# The span is a milestone
if ($from == $to) {
$span->milestone(1);
@@ -41,7 +47,13 @@
unless (defined $start) {
$start = $self->range->after($span->o_start);
- return unless defined $start;
+
+ unless (defined $start) {
+ if (DEBUG) {
+ warn $span->id . ' has no valid start';
+ };
+ return;
+ };
};
# Set start token position to span
@@ -57,25 +69,49 @@
unless (defined $end) {
$end = $self->range->before($span->o_end);
- return unless defined $end;
+
+ if (DEBUG && $span->o_end == 196) {
+ warn 'SPAN ends at ' . $span->o_end . ' and has ' . $end;
+ };
+
+ unless (defined $end) {
+ if (DEBUG) {
+ warn $span->id . ' has no valid end';
+ };
+ return;
+ };
# The next token of end has a character
- # offset AFTER th given end character offset
+ # offset AFTER the given end character offset
my $real_start = $self->stream->pos($end)->o_start;
# Ignore non-milestone elements outside the token stream!
- return if $to <= $real_start;
+ if ($to <= $real_start) {
+ if (DEBUG) {
+ warn 'Ignore ' . $span->id . ' is a non-milestone element outside the token stream';
+ };
+ return;
+ };
};
# $span->p_end($end);
# return unless $span->p_end >= $span->p_start;
# EXPERIMENTAL:
- return unless $end >= $span->p_start;
+ unless ($end >= $span->p_start) {
+ if (DEBUG) {
+ warn 'Ignore ' . $span->id . ' with ' . $span->p_start . '-' . $end;
+ };
+ return;
+ };
$span->p_end($end + 1);
}
+ if (DEBUG && $from == 124) {
+ warn 'exact: ' . $span->p_start . '-' . $span->p_end;
+ };
+
$span->hash($s) if $s;
$span;
@@ -116,6 +152,8 @@
$from = $p->bytes2chars($from);
$to = $p->bytes2chars($to);
}
+
+ # This is legacy treating of bytes2chars
elsif ($self->encoding eq 'xip') {
$from = $p->xip2chars($from);
$to = $p->xip2chars($to);
diff --git a/t/annotation/base_paragraphs.t b/t/annotation/base_paragraphs.t
index e5d0c6b..9f02f27 100644
--- a/t/annotation/base_paragraphs.t
+++ b/t/annotation/base_paragraphs.t
@@ -17,8 +17,8 @@
like($data->{foundries}, qr!base/paragraphs!, 'data');
is($data->{stream}->[0]->[0], '-:base/paragraphs$<i>1', 'Number of paragraphs');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text');
-is($data->{stream}->[0]->[3], '<>:base/s:p$<b>64<i>0<i>129<i>17<b>1', 'Paragraph');
+is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text');
+is($data->{stream}->[0]->[3], '<>:base/s:p$<b>64<i>0<i>129<i>18<b>1', 'Paragraph');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
done_testing;
diff --git a/t/annotation/base_sentences.t b/t/annotation/base_sentences.t
index 69bb313..8096e5b 100644
--- a/t/annotation/base_sentences.t
+++ b/t/annotation/base_sentences.t
@@ -17,8 +17,8 @@
like($data->{foundries}, qr!base/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:base/sentences$<i>1', 'Number of paragraphs');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text');
-is($data->{stream}->[0]->[3], '<>:base/s:s$<b>64<i>0<i>129<i>17<b>2', 'Sentence');
+is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text');
+is($data->{stream}->[0]->[3], '<>:base/s:s$<b>64<i>0<i>129<i>18<b>2', 'Sentence');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
done_testing;
diff --git a/t/annotation/connexor_morpho.t b/t/annotation/connexor_morpho.t
index de0f704..d426571 100644
--- a/t/annotation/connexor_morpho.t
+++ b/t/annotation/connexor_morpho.t
@@ -14,7 +14,7 @@
my $data = $tokens->to_data->{data};
like($data->{foundries}, qr!connexor/morpho!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[2], '_0$<i>0<i>3', 'Position');
is($data->{stream}->[1]->[1], 'cnx/l:letzt', 'Lemma');
is($data->{stream}->[1]->[2], 'cnx/p:A', 'POS');
diff --git a/t/annotation/connexor_phrase.t b/t/annotation/connexor_phrase.t
index a2cf6e3..f3df759 100644
--- a/t/annotation/connexor_phrase.t
+++ b/t/annotation/connexor_phrase.t
@@ -15,7 +15,7 @@
my $data = $tokens->to_data->{data};
like($data->{foundries}, qr!connexor/phrase!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[1]->[0], '<>:cnx/c:np$<b>64<i>4<i>30<i>4<b>0', 'Noun phrase');
done_testing;
diff --git a/t/annotation/connexor_sentences.t b/t/annotation/connexor_sentences.t
index 0c2891c..37db3fb 100644
--- a/t/annotation/connexor_sentences.t
+++ b/t/annotation/connexor_sentences.t
@@ -42,8 +42,8 @@
like($data->{foundries}, qr!connexor/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:cnx/sentences$<i>1', 'Number of paragraphs');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
-is($data->{stream}->[0]->[3], '<>:cnx/s:s$<b>64<i>0<i>129<i>17<b>0', 'Sentence');
+is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
+is($data->{stream}->[0]->[3], '<>:cnx/s:s$<b>64<i>0<i>129<i>18<b>0', 'Sentence');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
done_testing;
diff --git a/t/annotation/corenlp_morpho.t b/t/annotation/corenlp_morpho.t
index ec6e518..cefc94e 100644
--- a/t/annotation/corenlp_morpho.t
+++ b/t/annotation/corenlp_morpho.t
@@ -15,7 +15,7 @@
my $data = $tokens->to_data->{data};
like($data->{foundries}, qr!corenlp/morpho!, 'data');
like($data->{layerInfos}, qr!corenlp/p=tokens!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[3], 'corenlp/p:APPRART', 'POS');
is($data->{stream}->[1]->[1], 'corenlp/p:ADJ', 'POS');
is($data->{stream}->[2]->[1], 'corenlp/p:ADJA', 'POS');
diff --git a/t/annotation/corenlp_sentences.t b/t/annotation/corenlp_sentences.t
index b9e9b75..09611c6 100644
--- a/t/annotation/corenlp_sentences.t
+++ b/t/annotation/corenlp_sentences.t
@@ -17,8 +17,8 @@
like($data->{foundries}, qr!corenlp/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:corenlp/sentences$<i>1', 'Number of paragraphs');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
-is($data->{stream}->[0]->[3], '<>:corenlp/s:s$<b>64<i>0<i>129<i>17<b>0', 'Text');
+is($data->{stream}->[0]->[2], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
+is($data->{stream}->[0]->[3], '<>:corenlp/s:s$<b>64<i>0<i>129<i>18<b>0', 'Text');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
is($data->{stream}->[-1]->[0], '_17$<i>124<i>128', 'Position');
diff --git a/t/annotation/dereko_struct.t b/t/annotation/dereko_struct.t
index 5569369..921b58e 100644
--- a/t/annotation/dereko_struct.t
+++ b/t/annotation/dereko_struct.t
@@ -23,7 +23,7 @@
'Empty element');
-is($data->{stream}->[0]->[5], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[5], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
# Attributes:
is($data->{stream}->[0]->[11],
@@ -42,19 +42,19 @@
'@:dereko/s:pattern:text$<b>17<s>2',
'Attribute of idsHeader');
-is($data->{stream}->[4]->[1],
+is($data->{stream}->[5]->[1],
'<>:dereko/s:s$<b>64<i>32<i>42<i>6<b>6<s>1',
'Sentence span');
-is($data->{stream}->[4]->[2],
+is($data->{stream}->[5]->[2],
'@:dereko/s:broken:no$<b>17<s>1<i>6',
'Attribute of sentence span');
-is($data->{stream}->[6]->[0],
- '<>:dereko/s:pb$<b>65<i>42<i>42<i>6<b>6<s>1',
+is($data->{stream}->[7]->[0],
+ '<>:dereko/s:pb$<b>65<i>42<i>42<i>7<b>6<s>1',
'Pagebreak element');
-is($data->{stream}->[6]->[-1],
+is($data->{stream}->[7]->[-1],
'~:base/s:pb$<i>2<i>42',
'Pagebreak element');
diff --git a/t/annotation/glemm_morpho.t b/t/annotation/glemm_morpho.t
index 208d9d1..0ac9705 100644
--- a/t/annotation/glemm_morpho.t
+++ b/t/annotation/glemm_morpho.t
@@ -17,7 +17,7 @@
like($data->{foundries}, qr!glemm/morpho!, 'data');
like($data->{layerInfos}, qr!glemm/l=tokens!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[3], 'glemm/l:__zu', 'Lemma');
is($data->{stream}->[1]->[1], 'glemm/l:__letzt-', 'Lemma');
diff --git a/t/annotation/malt_dependency.t b/t/annotation/malt_dependency.t
index f58204f..515ae8f 100644
--- a/t/annotation/malt_dependency.t
+++ b/t/annotation/malt_dependency.t
@@ -51,7 +51,7 @@
is($stream->[0]->[5], '>:malt/d:ROOT$<b>33<i>0<i>49<i>0<i>6', 'Term2Term relation');
# Text element
-is($stream->[0]->[4], '<>:base/s:t$<b>64<i>0<i>238<i>30<b>0', 'Text element');
+is($stream->[0]->[4], '<>:base/s:t$<b>64<i>0<i>238<i>31<b>0', 'Text element');
done_testing;
__END__
diff --git a/t/annotation/mate_morpho.t b/t/annotation/mate_morpho.t
index 736fd31..9798ba0 100644
--- a/t/annotation/mate_morpho.t
+++ b/t/annotation/mate_morpho.t
@@ -19,7 +19,7 @@
like($data->{layerInfos}, qr!mate/l=tokens!, 'data');
like($data->{layerInfos}, qr!mate/m=tokens!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[4], 'mate/l:zu', 'POS');
is($data->{stream}->[0]->[5], 'mate/m:case:dat', 'POS');
is($data->{stream}->[0]->[6], 'mate/m:gender:neut', 'POS');
diff --git a/t/annotation/mate_morpho_attr.t b/t/annotation/mate_morpho_attr.t
index b32c12c..e556d0f 100644
--- a/t/annotation/mate_morpho_attr.t
+++ b/t/annotation/mate_morpho_attr.t
@@ -18,7 +18,7 @@
like($data->{layerInfos}, qr!mate/p=tokens!, 'data');
like($data->{layerInfos}, qr!mate/l=tokens!, 'data');
-is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Text boundary');
+is($data->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Text boundary');
is($data->{stream}->[0]->[2], '@:gender=neut$<b>16<s>1', 'POS');
is($data->{stream}->[0]->[3], '@:number=sg$<b>16<s>1', 'POS');
is($data->{stream}->[0]->[4], '@:case=dat$<b>16<s>1', 'POS');
diff --git a/t/annotation/mdp_dependency.t b/t/annotation/mdp_dependency.t
index de9144c..2fd341f 100644
--- a/t/annotation/mdp_dependency.t
+++ b/t/annotation/mdp_dependency.t
@@ -82,8 +82,8 @@
is($stream->[0]->[10], '>:mdp/d:ROOT$<b>33<i>0<i>317<i>0<i>40', 'Term-to-Element');
-is($stream->[-1]->[0], '>:mdp/d:ROOT$<b>33<i>26130<i>26153<i>3553<i>3554', 'Term-to-Element');
-is($stream->[3553]->[1], '<:mdp/d:ROOT$<b>34<i>26130<i>26153<i>3554<i>3553', 'Element-to-Term');
+is($stream->[-1]->[0], '>:mdp/d:ROOT$<b>33<i>26130<i>26153<i>3553<i>3555', 'Term-to-Element');
+is($stream->[3553]->[1], '<:mdp/d:ROOT$<b>34<i>26130<i>26153<i>3555<i>3553', 'Element-to-Term');
done_testing;
__END__
diff --git a/t/annotation/opennlp_morpho.t b/t/annotation/opennlp_morpho.t
index c8bcbb7..64d9008 100644
--- a/t/annotation/opennlp_morpho.t
+++ b/t/annotation/opennlp_morpho.t
@@ -10,7 +10,7 @@
ok(my $tokens = TestInit::tokens('0001'), 'Parse tokens');
-is($tokens->stream->pos(0)->to_string, '[(0-3)-:tokens$<i>18|<>:base/s:t$<b>64<i>0<i>129<i>17<b>0|_0$<i>0<i>3|i:zum|s:Zum]', 'Token is correct');
+is($tokens->stream->pos(0)->to_string, '[(0-3)-:tokens$<i>18|<>:base/s:t$<b>64<i>0<i>129<i>18<b>0|_0$<i>0<i>3|i:zum|s:Zum]', 'Token is correct');
is($tokens->stream->pos(1)->to_string, '[(4-11)_1$<i>4<i>11|i:letzten|s:letzten]', 'Token is correct');
diff --git a/t/annotation/opennlp_sentences.t b/t/annotation/opennlp_sentences.t
index 16c8a65..02eac83 100644
--- a/t/annotation/opennlp_sentences.t
+++ b/t/annotation/opennlp_sentences.t
@@ -17,7 +17,7 @@
like($data->{foundries}, qr!opennlp/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:opennlp/sentences$<i>1', 'Number of Sentences');
is($data->{stream}->[0]->[1], '-:tokens$<i>18', 'Number of tokens');
-is($data->{stream}->[0]->[3], '<>:opennlp/s:s$<b>64<i>0<i>129<i>17<b>0', 'Sentence');
+is($data->{stream}->[0]->[3], '<>:opennlp/s:s$<b>64<i>0<i>129<i>18<b>0', 'Sentence');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
done_testing;
diff --git a/t/annotation/tt_sentences.t b/t/annotation/tt_sentences.t
index 703f1a6..a7894b8 100644
--- a/t/annotation/tt_sentences.t
+++ b/t/annotation/tt_sentences.t
@@ -19,7 +19,7 @@
like($data->{foundries}, qr!treetagger/sentences!, 'data');
is($data->{stream}->[0]->[0], '-:tokens$<i>18', 'Number of tokens');
is($data->{stream}->[0]->[1], '-:tt/sentences$<i>1', 'Number of paragraphs');
-is($data->{stream}->[0]->[3], '<>:tt/s:s$<b>64<i>0<i>130<i>17<b>0', 'Text');
+is($data->{stream}->[0]->[3], '<>:tt/s:s$<b>64<i>0<i>130<i>18<b>0', 'Text');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
is($data->{stream}->[-1]->[0], '_17$<i>124<i>128', 'Position');
diff --git a/t/annotation/xip_constituency.t b/t/annotation/xip_constituency.t
index 026321f..0f8a0b8 100644
--- a/t/annotation/xip_constituency.t
+++ b/t/annotation/xip_constituency.t
@@ -19,8 +19,8 @@
# The length includes the punct - but that doesn't matter
is($data->{stream}->[0]->[1], '<>:xip/c:PREP$<b>64<i>0<i>3<i>1<b>3', 'Prep phrase');
is($data->{stream}->[0]->[2], '<>:xip/c:PP$<b>64<i>0<i>30<i>4<b>2', 'pp phrase');
-is($data->{stream}->[0]->[4], '<>:xip/c:TOP$<b>64<i>0<i>129<i>17<b>0', 'top phrase');
-is($data->{stream}->[0]->[5], '<>:xip/c:MC$<b>64<i>0<i>129<i>17<b>1', 'mc phrase');
+is($data->{stream}->[0]->[4], '<>:xip/c:TOP$<b>64<i>0<i>129<i>18<b>0', 'top phrase');
+is($data->{stream}->[0]->[5], '<>:xip/c:MC$<b>64<i>0<i>129<i>18<b>1', 'mc phrase');
is($data->{stream}->[-1]->[0], '<>:xip/c:VERB$<b>64<i>124<i>128<i>18<b>4', 'Noun phrase');
diff --git a/t/annotation/xip_sentences.t b/t/annotation/xip_sentences.t
index 3f2c62c..88b3e21 100644
--- a/t/annotation/xip_sentences.t
+++ b/t/annotation/xip_sentences.t
@@ -18,7 +18,7 @@
is($data->{stream}->[0]->[0], '-:tokens$<i>18', 'Number of tokens');
is($data->{stream}->[0]->[1], '-:xip/sentences$<i>1', 'Number of paragraphs');
-is($data->{stream}->[0]->[3], '<>:xip/s:s$<b>64<i>0<i>129<i>17<b>0', 'Text');
+is($data->{stream}->[0]->[3], '<>:xip/s:s$<b>64<i>0<i>129<i>18<b>0', 'Text');
is($data->{stream}->[0]->[4], '_0$<i>0<i>3', 'Position');
is($data->{stream}->[-1]->[0], '_17$<i>124<i>128', 'Position');
diff --git a/t/batch_file.t b/t/batch_file.t
index ed32401..55c4d2c 100644
--- a/t/batch_file.t
+++ b/t/batch_file.t
@@ -35,7 +35,7 @@
is($json->{data}->{foundries}, '', 'Foundries');
like($json->{data}->{text}, qr/^Zum letzten kulturellen/, 'Foundries');
is($json->{data}->{stream}->[0]->[0], '-:tokens$<i>18', 'Tokens');
-is($json->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Data');
+is($json->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Data');
# Generate with Gzip
$bf->{gzip} = 1;
@@ -56,7 +56,7 @@
is($json->{data}->{foundries}, '', 'Foundries');
like($json->{data}->{text}, qr/^Zum letzten kulturellen/, 'Foundries');
is($json->{data}->{stream}->[0]->[0], '-:tokens$<i>18', 'Tokens');
-is($json->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'Data');
+is($json->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'Data');
# Generate with annotations
$bf->{gzip} = 0;
@@ -80,7 +80,7 @@
like($json->{data}->{text}, qr/Ende Schuljahr eingestellt wird\.$/, 'Primary text');
-is($token->[1], '<>:base/s:t$<b>64<i>0<i>129<i>17<b>0', 'base/s');
+is($token->[1], '<>:base/s:t$<b>64<i>0<i>129<i>18<b>0', 'base/s');
is($token->[2], '_0$<i>0<i>3', 'position');
is($token->[3], 'corenlp/p:APPRART', 'corenlp');
is($token->[5], 'opennlp/p:APPRART', 'opennlp');
diff --git a/t/corpus/REDEW/DOC1/00000/data.xml b/t/corpus/REDEW/DOC1/00000/data.xml
new file mode 100644
index 0000000..3b85204
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/data.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="text.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<raw_text docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP">
+ <metadata file="metadata.xml" />
+ <text>Cechov,_Anton_Pavlovic_Gram.tg4_1.xml 1886 1880 Gram Čechov, Anton Pavlovič yes yes Erzähltext digbib no third chevron ... Hörst du, du alter Drachen? Oder machst du dir nichts aus unseren Worten?«</text>
+</raw_text>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml b/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml
new file mode 100644
index 0000000..66f7ba1
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml
@@ -0,0 +1,205 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+ <spanList>
+ <span id="s0" from="119" to="122" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">--</f>
+ <f name="pos">$(</f>
+ <f name="msd">msd=rfpos=SYM.Pun.Cont|sentstart=yes|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s1" from="123" to="128" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">Hörst</f>
+ <f name="pos">VVFIN</f>
+ <f name="msd">msd=rfpos=VFIN.Full.2.Sg.Pres.Ind|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s2" from="129" to="131" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">du</f>
+ <f name="pos">PPER</f>
+ <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s3" from="131" to="132" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">--</f>
+ <f name="pos">$,</f>
+ <f name="msd">msd=rfpos=SYM.Pun.Comma|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s4" from="133" to="135" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">du</f>
+ <f name="pos">PPER</f>
+ <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s5" from="136" to="141" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">alt</f>
+ <f name="pos">ADJA</f>
+ <f name="msd">msd=rfpos=ADJA.Pos.Gen.Pl.Masc|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s6" from="142" to="149" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">Drache</f>
+ <f name="pos">NN</f>
+ <f name="msd">msd=rfpos=N.Reg.Gen.Pl.Masc|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s7" from="149" to="150" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">--</f>
+ <f name="pos">$.</f>
+ <f name="msd">msd=rfpos=SYM.Pun.Sent|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s8" from="151" to="155" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">oder</f>
+ <f name="pos">NE</f>
+ <f name="msd">msd=rfpos=CONJ.Coord.-|sentstart=yes|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s9" from="156" to="162" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">machen</f>
+ <f name="pos">VVFIN</f>
+ <f name="msd">msd=rfpos=VFIN.Full.2.Sg.Pres.Ind|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s10" from="163" to="165" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">du</f>
+ <f name="pos">PPER</f>
+ <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s11" from="166" to="169" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">dir</f>
+ <f name="pos">PPER</f>
+ <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Dat.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s12" from="170" to="176" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">nichts</f>
+ <f name="pos">PIS</f>
+ <f name="msd">msd=rfpos=PRO.Indef.Subst.-.*.*.Neut|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s13" from="177" to="180" l="6">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">aus</f>
+ <f name="pos">APPR</f>
+ <f name="msd">msd=rfpos=APPR.Dat|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s14" from="181" to="188" l="7">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">unser</f>
+ <f name="pos">PPOSAT</f>
+ <f name="msd">msd=rfpos=PRO.Poss.Attr.-.Dat.Pl.Neut|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s15" from="189" to="195" l="7">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">Wort</f>
+ <f name="pos">NN</f>
+ <f name="msd">msd=rfpos=N.Reg.Dat.Pl.Neut|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s16" from="195" to="196" l="6">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">--</f>
+ <f name="pos">$.</f>
+ <f name="msd">msd=rfpos=SYM.Pun.Sent|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s17" from="196" to="197" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">"</f>
+ <f name="pos">$(</f>
+ <f name="msd">msd=rfpos=N.Reg.Nom.Sg.Masc|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/header.xml b/t/corpus/REDEW/DOC1/00000/header.xml
new file mode 100644
index 0000000..9738cd3
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/header.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="text" pattern="text" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+ <title>Gram</title>
+<!--- missing in origin data -->
+<textSigle>REDEW/DOC1.00000</textSigle>
+<!-- -->
+ <author>Čechov, Anton Pavlovič</author>
+ <respStmt>
+ <resp></resp>
+ <name></name>
+ </respStmt>
+ </titleStmt>
+ <publicationStmt>
+ <distributor>www.redewiedergabe.de</distributor>
+ </publicationStmt>
+ <sourceDesc>
+ <bibl></bibl>
+ </sourceDesc>
+ </fileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/struct/structure.xml b/t/corpus/REDEW/DOC1/00000/struct/structure.xml
new file mode 100644
index 0000000..c12f102
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/struct/structure.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+ <spanList>
+ <span id="s0" from="0" to="197" l="1">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">text</f>
+ </fs>
+ </span>
+ <span id="s1" from="0" to="118" l="2">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">fs</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="type">metadata</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s2" from="0" to="37" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">filename</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s3" from="38" to="42" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">year</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s4" from="43" to="47" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">decade</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s5" from="48" to="52" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">title</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s6" from="53" to="75" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">author</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s7" from="76" to="79" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">fictional</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s8" from="80" to="83" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">narrative</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s9" from="84" to="94" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">textype</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s10" from="95" to="101" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">source</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s11" from="102" to="104" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">dialect</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s12" from="105" to="110" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">perspective</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s13" from="111" to="118" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">quotes</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s14" from="118" to="197" l="2">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">body</f>
+ </fs>
+ </span>
+ <span id="s15" from="118" to="197" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">p</f>
+ </fs>
+ </span>
+ <span id="s16" from="118" to="197" l="4">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">said</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="id">1</f>
+ <f name="mode">direct</f>
+ <f name="content">speech</f>
+ <f name="level">1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s30" from="176" to="196" l="5">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">said</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="id">19</f>
+ <f name="mode">reported</f>
+ <f name="content">speech</f>
+ <f name="level">2</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s32" from="180" to="188" l="6">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">seg</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="id">19</f>
+ <f name="type">speaker</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s34" from="188" to="195" l="6">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">seg</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="id">19</f>
+ <f name="type">intExpr</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/header.xml b/t/corpus/REDEW/DOC1/header.xml
new file mode 100644
index 0000000..b64c4db
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/header.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="document" pattern="text" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+<!--- missing in origin data -->
+ <dokumentSigle>REDEW/DOC1</dokumentSigle>
+<!-- -->
+ <d.title>Redewiedergabe Dokument 1</d.title>
+ </titleStmt>
+ <publicationStmt>
+ <distributor/>
+ <pubAddress/>
+ <availability region="world">QAO-NC</availability>
+ <pubDate/>
+ </publicationStmt>
+ <sourceDesc>
+ <biblStruct>
+ <monogr>
+ <h.title type="main"/>
+ <edition>
+ <further/>
+ <kind/>
+ <appearance/>
+ </edition>
+ <imprint/>
+ <biblNote n="1">?</biblNote>
+ </monogr>
+ </biblStruct>
+ </sourceDesc>
+ </fileDesc>
+ <profileDesc>
+ <creation>
+ <creatDate>2020</creatDate>
+ <creatRef>?</creatRef>
+ <creatRefShort>?</creatRefShort>
+ </creation>
+ <textDesc>
+ <textType>?</textType>
+ <textTypeRef/>
+ </textDesc>
+ </profileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/corpus/REDEW/header.xml b/t/corpus/REDEW/header.xml
new file mode 100644
index 0000000..dc35127
--- /dev/null
+++ b/t/corpus/REDEW/header.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="corpus" pattern="allesaußerZtg/Zschr" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+<!--- missing in origin data -->
+ <korpusSigle>REDEW</korpusSigle>
+<!-- -->
+ <c.title>Redewiedergabe</c.title>
+ </titleStmt>
+ <publicationStmt>
+ <distributor> Institut für Deutsche Sprache </distributor>
+ <pubAddress> Postfach 10 16 21, D-68016 Mannheim </pubAddress>
+ <telephone> +49 (0)621 1581 0 </telephone>
+ <availability region="world">QAO-NC</availability>
+ <pubDate/>
+ </publicationStmt>
+ <sourceDesc>
+ <biblFull>
+ <titleStmt>
+ <x.title/>
+ </titleStmt>
+ <editionStmt/>
+ <publicationStmt>
+ <distributor/>
+ <pubAddress/>
+ <availability region="world">QAO-NC</availability>
+ <pubDate/>
+ </publicationStmt>
+ </biblFull>
+ <biblStruct>
+ <monogr>
+ <h.title type="main">Redewiedergabe</h.title>
+ <h.title type="sub"/>
+ <h.author>...</h.author>
+ <editor>...</editor>
+ <edition>
+ <further/>
+ <kind></kind>
+ <appearance/>
+ </edition>
+ <imprint>
+ <publisher>...</publisher>
+ <pubDate type="year">2020</pubDate>
+ <pubDate type="month"/>
+ <pubDate type="day"/>
+ <pubPlace key="DE"></pubPlace>
+ </imprint>
+ </monogr>
+ </biblStruct>
+ <reference type="super" assemblage="non-automatic">...</reference>
+ </sourceDesc>
+ </fileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/range.t b/t/range.t
index 27c7970..2091661 100644
--- a/t/range.t
+++ b/t/range.t
@@ -44,8 +44,8 @@
is($range->after(0), 0, 'After is fine');
is($range->after(1), 0, 'After is fine');
is($range->after(2), 0, 'After is fine');
-is($range->after(3), 0, 'After is fine');
-is($range->after(14), 0, 'After is fine');
+is($range->after(3), 1, 'After is fine');
+is($range->after(14), 1, 'After is fine');
is($range->after(15), 1, 'After is fine');
is($range->to_string,
diff --git a/t/real/agd.t b/t/real/agd.t
index 6ba7ebe..7745bea 100644
--- a/t/real/agd.t
+++ b/t/real/agd.t
@@ -118,12 +118,12 @@
$first_token = join('||', @{$output->{data}->{stream}->[0]});
like($first_token, qr!<>:base/s:s\$<b>64<i>0<i>16<i>2<b>1!);
-my $token = join('||', @{$output->{data}->{stream}->[1]});
+my $token = join('||', @{$output->{data}->{stream}->[2]});
like($token, qr!<>:base/s:s\$<b>64<i>16<i>23<i>4<b>1!);
-$token = join('||', @{$output->{data}->{stream}->[2]});
+$token = join('||', @{$output->{data}->{stream}->[3]});
unlike($token, qr!<>:base/s:s!);
-$token = join('||', @{$output->{data}->{stream}->[3]});
+$token = join('||', @{$output->{data}->{stream}->[4]});
like($token, qr!<>:base/s:s\$<b>64<i>23<i>27<i>5<b>1!);
$token = join('||', @{$output->{data}->{stream}->[5]});
diff --git a/t/real/goethe-2.t b/t/real/goethe-2.t
index ff3cba4..691889d 100644
--- a/t/real/goethe-2.t
+++ b/t/real/goethe-2.t
@@ -138,7 +138,7 @@
like($first_token, qr/s:Autobiographische/, 'data');
like($first_token, qr/_0\$<i>0<i>17/, 'data');
like($first_token, qr!<>:dereko/s:s\$<b>64<i>0<i>30<i>2<b>4!, 'data');
-like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35242<i>5233<b>0!, 'data');
+like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35242<i>5234<b>0!, 'data');
# like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35250<i>5233<b>0!, 'data');
like($first_token, qr!<>:base/s:s\$<b>64<i>0<i>30<i>2<b>2!, 'data');
like($first_token, qr!-:base\/paragraphs\$\<i\>14!, 'data');
diff --git a/t/real/goethe.t b/t/real/goethe.t
index 5bc5eb2..03af7cc 100644
--- a/t/real/goethe.t
+++ b/t/real/goethe.t
@@ -138,7 +138,7 @@
like($first_token, qr/s:Autobiographische/, 'data');
like($first_token, qr/_0\$<i>0<i>17/, 'data');
like($first_token, qr!<>:base/s:s\$<b>64<i>0<i>30<i>2<b>2!, 'data');
-like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35199<i>5226<b>0!, 'data');
+like($first_token, qr!<>:base\/s:t\$<b>64<i>0<i>35199<i>5227<b>0!, 'data');
## OpenNLP
$tokens->add('OpenNLP', 'Sentences');
diff --git a/t/real/hnc.t b/t/real/hnc.t
index 9aca507..5e44f6b 100644
--- a/t/real/hnc.t
+++ b/t/real/hnc.t
@@ -88,8 +88,9 @@
my $output = decode_json( $tokens->to_json );
-is($output->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>4368<i>577<b>0', 't');
+is($output->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>4368<i>578<b>0', 't');
is($output->{data}->{stream}->[0]->[3], 'i:addbot', 't');
+is($output->{data}->{stream}->[-1]->[0], '_577$<i>4359<i>4368', 't');
## Base
diff --git a/t/real/redew.t b/t/real/redew.t
new file mode 100644
index 0000000..a5fe45b
--- /dev/null
+++ b/t/real/redew.t
@@ -0,0 +1,155 @@
+use strict;
+use warnings;
+use Test::More;
+use Data::Dumper;
+use JSON::XS;
+use Log::Log4perl;
+
+use utf8;
+use lib 'lib', '../lib';
+
+use File::Basename 'dirname';
+use File::Spec::Functions 'catdir';
+
+# Initialize log4perl object
+Log::Log4perl->init({
+ 'log4perl.rootLogger' => 'ERROR, STDERR',
+ 'log4perl.appender.STDERR' => 'Log::Log4perl::Appender::ScreenColoredLevels',
+ 'log4perl.appender.STDERR.layout' => 'PatternLayout',
+ 'log4perl.appender.STDERR.layout.ConversionPattern' => '[%r] %F %L %c - %m%n'
+});
+
+use_ok('KorAP::XML::Krill');
+
+my $path = catdir(dirname(__FILE__), '../corpus/REDEW/DOC1/00000');
+
+ok(my $doc = KorAP::XML::Krill->new( path => $path . '/' ), 'Load Korap::Document');
+ok($doc->parse, 'Parse document');
+
+is($doc->text_sigle, 'REDEW/DOC1/00000', 'Correct text sigle');
+is($doc->doc_sigle, 'REDEW/DOC1', 'Correct document sigle');
+is($doc->corpus_sigle, 'REDEW', 'Correct corpus sigle');
+
+my $meta = $doc->meta;
+ok(!$meta->{T_title}, 'Title'); # ???
+ok(!$meta->{T_sub_title}, 'SubTitle');
+ok(!$meta->{T_author}, 'Author');
+ok(!$meta->{A_editor}, 'Editor');
+ok(!$meta->{S_pub_place}, 'PubPlace');
+is($meta->{A_publisher}, '...', 'Publisher'); # ???
+
+is($meta->{S_text_type}, '?', 'Text Type'); # ???
+ok(!$meta->{S_text_type_art}, 'No Text Type Art');
+ok(!$meta->{S_text_type_ref}, 'No Text Type Ref');
+ok(!$meta->{S_text_domain}, 'No Text Domain');
+ok(!$meta->{S_text_column}, 'No Text Column');
+
+ok(!$meta->{K_text_class}->[0], 'Correct Text Class');
+
+is($meta->{D_pub_date}, '20200000', 'Creation date');
+is($meta->{D_creation_date}, '20200000', 'Creation date');
+is($meta->{S_availability}, 'QAO-NC', 'License'); # ???
+ok(!$meta->{A_pages}, 'Pages');
+
+ok(!$meta->{A_file_edition_statement}, 'File Statement');
+ok(!$meta->{A_bibl_edition_statement}, 'Bibl Statement');
+
+ok(!$meta->{A_reference}, 'Reference');
+ok(!$meta->{S_language}, 'Language'); # ???
+
+is($meta->{T_corpus_title}, 'Redewiedergabe', 'Correct Corpus title');
+ok(!$meta->{T_corpus_sub_title}, 'Correct Corpus sub title');
+is($meta->{T_corpus_author}, '...', 'Correct Corpus author'); # ???
+is($meta->{A_corpus_editor}, '...', 'Correct Corpus editor'); # ???
+
+is($meta->{T_doc_title}, 'Redewiedergabe Dokument 1', 'Correct Doc title');
+ok(!$meta->{T_doc_sub_title}, 'Correct Doc sub title');
+ok(!$meta->{T_doc_author}, 'Correct Doc author');
+ok(!$meta->{A_doc_editor}, 'Correct doc editor');
+
+# Tokenization
+use_ok('KorAP::XML::Tokenizer');
+
+my ($token_base_foundry, $token_base_layer) = (qw/drukola Morpho/);
+
+# Get tokenization
+my $tokens = KorAP::XML::Tokenizer->new(
+ path => $doc->path,
+ doc => $doc,
+ foundry => $token_base_foundry,
+ layer => $token_base_layer,
+ name => 'tokens'
+);
+
+ok($tokens, 'Token Object is fine');
+ok($tokens->parse, 'Token parsing is fine');
+
+my $output = decode_json( $tokens->to_json );
+
+is(substr($output->{data}->{text}, 0, 100), 'Cechov,_Anton_Pavlovic_Gram.tg4_1.xml 1886 1880 Gram Čechov, Anton Pavlovič yes yes Erzähltext digbi', 'Primary Data');
+
+is($output->{data}->{name}, 'tokens', 'tokenName');
+is($output->{data}->{tokenSource}, 'drukola#morpho', 'tokenSource');
+is($output->{version}, '0.03', 'version');
+
+is($output->{data}->{foundries}, '', 'Foundries');
+is($output->{data}->{layerInfos}, '', 'layerInfos');
+is($output->{data}->{stream}->[0]->[4], 's:Hörst', 'data');
+
+is($output->{textSigle}, 'REDEW/DOC1/00000', 'Correct text sigle');
+is($output->{docSigle}, 'REDEW/DOC1', 'Correct document sigle');
+is($output->{corpusSigle}, 'REDEW', 'Correct corpus sigle');
+
+ok(!$output->{title}, 'Title');
+ok(!$output->{subTitle}, 'Correct SubTitle');
+ok(!$output->{author}, 'Author');
+ok(!exists $output->{editor}, 'Publisher');
+
+# Add annotations
+$tokens->add('DRuKoLa', 'Morpho');
+$tokens->add('DeReKo', 'Structure');
+
+$output = decode_json( $tokens->to_json );
+
+my $first = $output->{data}->{stream}->[0];
+
+is('-:tokens$<i>13',$first->[0]);
+is('<>:base/s:t$<b>64<i>0<i>197<i>13<b>0',$first->[1]);
+is('<>:dereko/s:text$<b>64<i>0<i>197<i>13<b>0',$first->[2]);
+is('<>:dereko/s:body$<b>64<i>118<i>197<i>13<b>1',$first->[3]);
+is('<>:dereko/s:p$<b>64<i>118<i>197<i>13<b>2',$first->[4]);
+is('<>:dereko/s:said$<b>64<i>118<i>197<i>13<b>3<s>1',$first->[5]);
+is('@:dereko/s:level:1$<b>17<s>1<i>13',$first->[6]);
+is('@:dereko/s:content:speech$<b>17<s>1<i>13',$first->[7]);
+is('@:dereko/s:mode:direct$<b>17<s>1<i>13',$first->[8]);
+is('@:dereko/s:id:1$<b>17<s>1<i>13',$first->[9]);
+is('_0$<i>123<i>128',$first->[10]);
+is("drukola/l:H\x{f6}rst",$first->[11]);
+is('drukola/m:msd:rfpos',$first->[12]);
+is('drukola/m:sentstart:no',$first->[13]);
+is('drukola/m:stwr:direct.speech.1',$first->[14]);
+is('drukola/p:VVFIN',$first->[15]);
+is("i:h\x{f6}rst",$first->[16]);
+is("s:H\x{f6}rst",$first->[17]);
+
+my $nine = join(',', @{$output->{data}->{stream}->[9]});
+like($nine, qr{drukola\/l:nichts}, 'Nichts');
+like($nine, qr{_9\$<i>170<i>176}, 'Term boundaries');
+unlike($nine, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>13<b>4<s>1}, 'Term boundaries');
+
+my $ten = join(',', @{$output->{data}->{stream}->[10]});
+like($ten, qr{_10\$<i>177<i>180}, 'Term boundaries');
+like($ten, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>13<b>4<s>1}, 'Term boundaries');
+
+my $eleven = join(',', @{$output->{data}->{stream}->[11]});
+like($eleven, qr{_11\$<i>181<i>188}, 'Term boundaries');
+like($eleven, qr{<>:dereko/s:seg\$<b>64<i>180<i>188<i>12<b>5<s>1}, 'Segment');
+
+
+my $twelve = join(',', @{$output->{data}->{stream}->[12]});
+like($twelve, qr{_12\$<i>189<i>195}, 'Term boundaries');
+like($twelve, qr{drukola/l:Wort}, 'Lemma');
+like($twelve, qr{<>:dereko/s:seg\$<b>64<i>188<i>195<i>13<b>5<s>1}, 'Segment');
+
+done_testing;
+__END__
diff --git a/t/real/rei.t b/t/real/rei.t
index b375c24..e08516d 100644
--- a/t/real/rei.t
+++ b/t/real/rei.t
@@ -209,7 +209,7 @@
is('<>:dereko/s:docTitle$<b>64<i>0<i>91<i>11<b>3', $first->[15]);
is('<>:dereko/s:titlePart$<b>64<i>0<i>91<i>11<b>4<s>2', $first->[16]);
is('<>:dereko/s:s$<b>64<i>0<i>91<i>11<b>5', $first->[17]);
-is('<>:base/s:t$<b>64<i>0<i>17859<i>2640<b>0', $first->[18]);
+is('<>:base/s:t$<b>64<i>0<i>17859<i>2641<b>0', $first->[18]);
is('>:malt/d:ROOT$<b>33<i>0<i>48<i>0<i>7', $first->[19]);
is('<:malt/d:PP$<b>32<i>1', $first->[20]);
is('<:malt/d:ROOT$<b>34<i>0<i>48<i>7<i>0', $first->[21]);
@@ -225,16 +225,17 @@
my $last = $output->{data}->{stream}->[-1];
-is('<>:dereko/s:text$<b>65<i>17859<i>17859<i>2640<b>0', $last->[0]);
-is('<>:dereko/s:back$<b>65<i>17859<i>17859<i>2640<b>1', $last->[1]);
-is('>:malt/d:APP$<b>32<i>2639', $last->[2]);
-is('_2640$<i>17851<i>17859', $last->[3]);
-is('corenlp/p:NE', $last->[4]);
-is("i:schr\x{f6}der", $last->[5]);
-is('opennlp/p:NE', $last->[6]);
-is("s:Schr\x{f6}der", $last->[7]);
-is("tt/l:Schr\x{f6}der", $last->[8]);
-is('tt/p:NE', $last->[9]);
+# Milestones behind the final token are no longer indexed
+# is('<>:dereko/s:text$<b>65<i>17859<i>17859<i>2640<b>0', $last->[0]);
+# is('<>:dereko/s:back$<b>65<i>17859<i>17859<i>2640<b>1', $last->[1]);
+is('>:malt/d:APP$<b>32<i>2639', $last->[0]);
+is('_2640$<i>17851<i>17859', $last->[1]);
+is('corenlp/p:NE', $last->[2]);
+is("i:schr\x{f6}der", $last->[3]);
+is('opennlp/p:NE', $last->[4]);
+is("s:Schr\x{f6}der", $last->[5]);
+is("tt/l:Schr\x{f6}der", $last->[6]);
+is('tt/p:NE', $last->[7]);
# REI/BNG/00071
@@ -282,7 +283,7 @@
is('-:corenlp/sentences$<i>69', $first->[2]);
is('-:opennlp/sentences$<i>65', $first->[3]);
is('-:tokens$<i>1009', $first->[4]);
-is('-:tt/sentences$<i>95', $first->[5]);
+is('-:tt/sentences$<i>94', $first->[5]);
is('<:malt/d:APP$<b>32<i>1', $first->[6]);
is('<>:corenlp/c:MPN$<b>64<i>0<i>16<i>2<b>3', $first->[7]);
is('<>:base/s:s$<b>64<i>0<i>48<i>5<b>2', $first->[8]);
@@ -297,8 +298,8 @@
is('<>:dereko/s:titlePage$<b>64<i>0<i>61<i>8<b>2<s>1', $first->[17]);
is('<>:dereko/s:docTitle$<b>64<i>0<i>61<i>8<b>3', $first->[18]);
is('<>:opennlp/s:s$<b>64<i>0<i>173<i>24<b>0', $first->[19]);
-is('<>:base/s:t$<b>64<i>0<i>7008<i>1008<b>0', $first->[20]);
-is('<>:dereko/s:text$<b>64<i>0<i>7008<i>1008<b>0', $first->[21]);
+is('<>:base/s:t$<b>64<i>0<i>7008<i>1009<b>0', $first->[20]);
+is('<>:dereko/s:text$<b>64<i>0<i>7008<i>1009<b>0', $first->[21]);
is('>:malt/d:GMOD$<b>32<i>3', $first->[22]);
is('<:malt/d:ROOT$<b>34<i>0<i>51<i>6<i>3', $first->[23]);
is('@:dereko/s:id:bng.00071-0-titlepage$<b>17<s>1<i>8', $first->[24]);
@@ -312,16 +313,17 @@
is('tt/p:NE', $first->[32]);
$last = $output->{data}->{stream}->[-1];
-is('<>:dereko/s:back$<b>65<i>7008<i>7008<i>1008<b>1', $last->[0]);
-is('<>:dereko/s:div$<b>65<i>7008<i>7008<i>1008<b>2<s>1', $last->[1]);
-is('@:dereko/s:n:1$<b>17<s>1', $last->[2]);
-is('@:dereko/s:type:footnotes$<b>17<s>1', $last->[3]);
-is('@:dereko/s:complete:y$<b>17<s>1', $last->[4]);
-is('_1008$<i>6990<i>7006', $last->[5]);
-is('corenlp/p:NN', $last->[6]);
-is('i:befreiungsschlag', $last->[7]);
-is('opennlp/p:NN', $last->[8]);
-is('s:Befreiungsschlag', $last->[9]);
+# No longer indexed:
+#is('<>:dereko/s:back$<b>65<i>7008<i>7008<i>1009<b>1', $last->[0]);
+#is('<>:dereko/s:div$<b>65<i>7008<i>7008<i>1009<b>2<s>1', $last->[1]);
+#is('@:dereko/s:n:1$<b>17<s>1', $last->[2]);
+#is('@:dereko/s:type:footnotes$<b>17<s>1', $last->[3]);
+#is('@:dereko/s:complete:y$<b>17<s>1', $last->[4]);
+is('_1008$<i>6990<i>7006', $last->[0]);
+is('corenlp/p:NN', $last->[1]);
+is('i:befreiungsschlag', $last->[2]);
+is('opennlp/p:NN', $last->[3]);
+is('s:Befreiungsschlag', $last->[4]);
done_testing;
__END__
diff --git a/t/real/wpd.t b/t/real/wpd.t
index 517f9be..e45db0a 100644
--- a/t/real/wpd.t
+++ b/t/real/wpd.t
@@ -78,10 +78,17 @@
my $stream = $tokens->to_data->{data}->{stream};
# This is not a goot relation example
-is($stream->[77]->[0],
- '<:mate/d:--$<b>34<i>498<i>499<i>78<i>78',
- 'element to term');
-is($stream->[78]->[0], '>:mate/d:--$<b>33<i>498<i>499<i>77<i>78', 'term to element');
+ is($stream->[79]->[0],
+ '>:mate/d:CJ$<b>32<i>68',
+ 'term to term');
+ is($stream->[79]->[1], '<:mate/d:PD$<b>32<i>81', 'term to term');
+
+
+# These are no longer aligned
+# is($stream->[77]->[0],
+# '<:mate/d:--$<b>34<i>498<i>499<i>78<i>78',
+# 'element to term');
+# is($stream->[78]->[0], '>:mate/d:--$<b>33<i>498<i>499<i>77<i>78', 'term to element');
$tokens->add('Base', 'Sentences');
diff --git a/t/real/wpf.t b/t/real/wpf.t
index 4a28c2f..ba6fb94 100644
--- a/t/real/wpf.t
+++ b/t/real/wpf.t
@@ -101,10 +101,11 @@
$tokens->add('Talismane', 'Morpho');
$stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[10], 'talismane/l:Sigmund', 'Talismane morpho annotation');
-is($stream->[1]->[11], 'talismane/m:g:m', 'Talismane morpho annotation');
-is($stream->[1]->[12], 'talismane/m:n:s', 'Talismane morpho annotation');
-is($stream->[1]->[13], 'talismane/p:NPP', 'Talismane morpho annotation');
+
+is($stream->[1]->[9], 'talismane/l:Sigmund', 'Talismane morpho annotation');
+is($stream->[1]->[10], 'talismane/m:g:m', 'Talismane morpho annotation');
+is($stream->[1]->[11], 'talismane/m:n:s', 'Talismane morpho annotation');
+is($stream->[1]->[12], 'talismane/p:NPP', 'Talismane morpho annotation');
is($stream->[300]->[5], 'talismane/l:son', 'Talismane lemma annotation');
is($stream->[300]->[6], 'talismane/m:g:m', 'Talismane morph annotation');
@@ -116,7 +117,10 @@
# Add Malt dependency
$tokens->add('Malt', 'Dependency');
$stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[1], '>:malt/d:dep$<b>33<i>7<i>8<i>0<i>1', 'Malt dep annotation');
+
+# This is no longer indexed
+# is($stream->[1]->[1], '>:malt/d:dep$<b>33<i>7<i>8<i>0<i>1', 'Malt dep annotation');
+
is($stream->[1]->[2], '<:malt/d:dep$<b>32<i>2', 'Malt dep annotation');
is($stream->[300]->[1], '>:malt/d:punct$<b>32<i>302', 'Malt dep annotation');
@@ -124,9 +128,10 @@
# Add TreeTagger morpho
$tokens->add('TreeTagger', 'Morpho');
$stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[14], 'tt/p:ADJ$<b>129<b>26', 'TreeTagger morph annotation');
-is($stream->[1]->[15], 'tt/p:NAM$<b>129<b>200', 'TreeTagger morph annotation');
-is($stream->[1]->[16], 'tt/p:NOM$<b>129<b>27', 'TreeTagger morph annotation');
+
+is($stream->[1]->[13], 'tt/p:ADJ$<b>129<b>26', 'TreeTagger morph annotation');
+is($stream->[1]->[14], 'tt/p:NAM$<b>129<b>200', 'TreeTagger morph annotation');
+is($stream->[1]->[15], 'tt/p:NOM$<b>129<b>27', 'TreeTagger morph annotation');
done_testing;
diff --git a/t/script/base.t b/t/script/base.t
index 08baa64..7bfe48d 100644
--- a/t/script/base.t
+++ b/t/script/base.t
@@ -58,7 +58,7 @@
is($token->[5], '<>:base/s:s$<b>64<i>0<i>30<i>2<b>2', 'struct');
is($token->[7], '<>:dereko/s:s$<b>64<i>0<i>30<i>2<b>4', 'struct');
-is($token->[8], '<>:base/s:t$<b>64<i>0<i>35242<i>5238<b>0', 'struct');
+is($token->[8], '<>:base/s:t$<b>64<i>0<i>35242<i>5239<b>0', 'struct');
$token = $stream->[4];
is($token->[0], '<>:base/s:s$<b>64<i>53<i>254<i>32<b>2', 'struct');
diff --git a/t/script/non_verbal_tokens.t b/t/script/non_verbal_tokens.t
index fa460e4..5327523 100644
--- a/t/script/non_verbal_tokens.t
+++ b/t/script/non_verbal_tokens.t
@@ -54,7 +54,7 @@
my $token = $stream->[4];
is($token->[3], 'dgd/l:pui', 'Token');
$token = $stream->[5];
-is($token->[13], 'dgd/l:xui', 'Token');
+is($token->[15], 'dgd/l:xui', 'Token');
$call = join(
' ',
@@ -88,10 +88,10 @@
is($token->[3], 'dgd/l:pui', 'Token');
$token = $stream->[5];
-is($token->[14], 'dgd/para:pause$<b>128<s>5', 'Token');
+is($token->[4], 'dgd/para:pause$<b>128<s>2', 'Token');
$token = $stream->[6];
-is($token->[1], 'dgd/l:xui', 'Token');
+is($token->[13], 'dgd/l:xui', 'Token');
diff --git a/t/script/non_word_tokens.t b/t/script/non_word_tokens.t
index 0eb9e14..b156866 100644
--- a/t/script/non_word_tokens.t
+++ b/t/script/non_word_tokens.t
@@ -51,7 +51,7 @@
is($json->{data}->{foundries}, 'base base/paragraphs base/sentences connexor connexor/morpho connexor/phrase connexor/sentences connexor/syntax corenlp corenlp/sentences dereko dereko/structure mate mate/dependency mate/morpho opennlp opennlp/morpho opennlp/sentences treetagger treetagger/morpho treetagger/sentences xip xip/constituency xip/morpho xip/sentences', 'Foundries');
my $stream = $json->{data}->{stream};
my $token = $stream->[12];
-is($token->[17], 's:Vokal', 'Token');
+is($token->[16], 's:Vokal', 'Token');
$token = $stream->[13];
is($token->[23], 's:Der', 'Token');
diff --git a/t/sgbr/base.t b/t/sgbr/base.t
index 718a0ba..073209e 100644
--- a/t/sgbr/base.t
+++ b/t/sgbr/base.t
@@ -32,8 +32,8 @@
is($stream->[0]->[0], '-:base/sentences$<i>1');
is($stream->[0]->[1], '-:tokens$<i>15');
-is($stream->[0]->[2], '<>:base/s:t$<b>64<i>0<i>115<i>14<b>0');
-is($stream->[0]->[3], '<>:base/s:s$<b>64<i>16<i>114<i>14<b>2');
+is($stream->[0]->[2], '<>:base/s:t$<b>64<i>0<i>115<i>15<b>0');
+is($stream->[0]->[3], '<>:base/s:s$<b>64<i>16<i>114<i>15<b>2');
is($stream->[0]->[4], '_0$<i>17<i>18');
done_testing;
diff --git a/t/sgbr/lemma.t b/t/sgbr/lemma.t
index 3221dba..6635175 100644
--- a/t/sgbr/lemma.t
+++ b/t/sgbr/lemma.t
@@ -32,7 +32,7 @@
my $stream = $data->{stream};
is($stream->[0]->[0], '-:tokens$<i>51', 'Token number');
-is($stream->[0]->[1], '<>:base/s:t$<b>64<i>0<i>365<i>50<b>0', 'Text Boundary');
+is($stream->[0]->[1], '<>:base/s:t$<b>64<i>0<i>365<i>51<b>0', 'Text Boundary');
is($stream->[0]->[2], '_0$<i>0<i>18', 'Position');
is($stream->[0]->[3], 'i:sommerüberraschung', 'First term');
is($stream->[0]->[4], 's:Sommerüberraschung', 'First term');
diff --git a/t/sgbr/pos.t b/t/sgbr/pos.t
index 0163ed1..d750cb1 100644
--- a/t/sgbr/pos.t
+++ b/t/sgbr/pos.t
@@ -33,7 +33,7 @@
my $stream = $data->{stream};
is($stream->[0]->[0], '-:tokens$<i>51', 'Token number');
-is($stream->[0]->[1], '<>:base/s:t$<b>64<i>0<i>365<i>50<b>0', 'Text boundary');
+is($stream->[0]->[1], '<>:base/s:t$<b>64<i>0<i>365<i>51<b>0', 'Text boundary');
is($stream->[0]->[2], '_0$<i>0<i>18', 'Position');
is($stream->[0]->[3], 'i:sommerüberraschung', 'First term');
is($stream->[0]->[4], 's:Sommerüberraschung', 'First term');
diff --git a/t/tokenization.t b/t/tokenization.t
index eecb1d7..da135e5 100644
--- a/t/tokenization.t
+++ b/t/tokenization.t
@@ -73,7 +73,8 @@
is($json->{data}->{name}, 'tokens');
is($json->{data}->{tokenSource}, 'opennlp#tokens');
-is($json->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>6083<i>1067<b>0');
+is($json->{data}->{stream}->[0]->[0], '-:tokens$<i>1068');
+is($json->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>6083<i>1068<b>0');
done_testing;
diff --git a/t/transform.t b/t/transform.t
index 4537bcb..c071f81 100644
--- a/t/transform.t
+++ b/t/transform.t
@@ -133,7 +133,7 @@
is_deeply(
_t2h($tokens->stream->pos(0)->to_string),
- _t2h('[(0-1)s:A|i:a|_0$<i>0<i>1|-:tokens$<i>923|mate/p:XY|<>:base/s:s$<b>64<i>0<i>74<i>13<b>2|<>:base/s:t$<b>64<i>0<i>6083<i>922<b>0|-:base/sentences$<i>96]'),
+ _t2h('[(0-1)s:A|i:a|_0$<i>0<i>1|-:tokens$<i>923|mate/p:XY|<>:base/s:s$<b>64<i>0<i>74<i>13<b>2|<>:base/s:t$<b>64<i>0<i>6083<i>923<b>0|-:base/sentences$<i>96]'),
'Startinfo'
);
@@ -146,7 +146,7 @@
'-:tokens$<i>923|'.
'mate/p:XY|'.
'<>:base/s:s$<b>64<i>0<i>74<i>13<b>2|'.
- '<>:base/s:t$<b>64<i>0<i>6083<i>922<b>0|'.
+ '<>:base/s:t$<b>64<i>0<i>6083<i>923<b>0|'.
'-:base/sentences$<i>96|'.
'<>:base/s:p$<b>64<i>0<i>224<i>34<b>1|'.
'-:base/paragraphs$<i>76|'.
@@ -154,7 +154,7 @@
'<>:opennlp/s:s$<b>64<i>0<i>74<i>13<b>0|'.
'-:opennlp/sentences$<i>50|'.
'<>:corenlp/s:s$<b>64<i>0<i>6<i>2<b>0|'.
- '-:corenlp/sentences$<i>68|'.
+ '-:corenlp/sentences$<i>67|'.
'cnx/l:A|'.
'cnx/p:N|'.
'cnx/syn:@NH|'.