Fixed span position offset bug and fixed milestones behind last token bug
Change-Id: I6250d515ec132e9024b1eaca82523887b7d6f643
diff --git a/Changes b/Changes
index 053e517..a351a92 100644
--- a/Changes
+++ b/Changes
@@ -1,5 +1,10 @@
0.40 2020-03-01
- Fixed XIP parser.
+ - Added example corpus of the
+ Redewiedergabe-Korpus.
+ - Fixed span offset bug.
+ - Fixed milestones behind the last
+ token bug.
0.39 2020-02-19
- Added Talismane support.
diff --git a/lib/KorAP/XML/Annotation/DGD/Structure.pm b/lib/KorAP/XML/Annotation/DGD/Structure.pm
index 6961b45..347b6c0 100644
--- a/lib/KorAP/XML/Annotation/DGD/Structure.pm
+++ b/lib/KorAP/XML/Annotation/DGD/Structure.pm
@@ -73,7 +73,7 @@
o_start => $o_start,
o_end => $_->[1],
p_start => $p_start,
- p_end => $_->[0] + 1,
+ p_end => $_->[0],
pti => 64
);
$mt->payload('<b>1');
diff --git a/lib/KorAP/XML/Annotation/DeReKo/Structure.pm b/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
index 769e629..cc2b751 100644
--- a/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
+++ b/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
@@ -36,6 +36,16 @@
# Get the mtt
my $mtt = $stream->pos($p_start);
+ unless ($mtt) {
+
+ # This is a special case were a milestone is at the
+ # end of a text and can't be indexed at the moment!
+ $$self->log->warn(
+ 'Span ' . $span->to_string . ' can\'t be indexed'
+ );
+ return;
+ };
+
my $p_end = $span->p_end;
# Add structure
diff --git a/lib/KorAP/XML/Annotation/Mate/Dependency.pm b/lib/KorAP/XML/Annotation/Mate/Dependency.pm
index ffc3223..6d242c9 100644
--- a/lib/KorAP/XML/Annotation/Mate/Dependency.pm
+++ b/lib/KorAP/XML/Annotation/Mate/Dependency.pm
@@ -30,126 +30,126 @@
# Iterate over relations
foreach (@$rel) {
- my $label = $_->{-label};
+ my $label = $_->{-label};
- # Relation type is unary
- # Unary means, it refers to itself!
- if ($_->{-type} && $_->{-type} eq 'unary') {
+ # Relation type is unary
+ # Unary means, it refers to itself!
+ if ($_->{-type} && $_->{-type} eq 'unary') {
- # I have no clue, what -- should mean
- # next if $_->{-label} eq '--';
+ # I have no clue, what -- should mean
+ # next if $_->{-label} eq '--';
- # Get target node - not very elegant
- # This is only necessary for nodes with attributes
- # my $target = $stream->get_node(
- # $source, 'mate/d:' . $NODE_LABEL
- # );
+ # Get target node - not very elegant
+ # This is only necessary for nodes with attributes
+ # my $target = $stream->get_node(
+ # $source, 'mate/d:' . $NODE_LABEL
+ # );
- # Target is at the same position!
- my $pos = $source->pos;
+ # Target is at the same position!
+ my $pos = $source->pos;
- my %rel = (
- pti => 32, # term-to-term relation
- payload =>
- '<i>' . $pos # . # right part token position
-# '<s>0' . # $target->tui . # left part tui
-# '<s>0' # . $target->tui # right part tui
- );
+ my %rel = (
+ pti => 32, # term-to-term relation
+ payload =>
+ '<i>' . $pos # . # right part token position
+ # '<s>0' . # $target->tui . # left part tui
+ # '<s>0' # . $target->tui # right part tui
+ );
- # Add relations
- $mtt->add(
- term => '>:mate/d:' . $label,
- %rel
- );
- $mtt->add(
- term => '<:mate/d:' . $label,
- %rel
- );
- }
+ # Add relations
+ $mtt->add(
+ term => '>:mate/d:' . $label,
+ %rel
+ );
+ $mtt->add(
+ term => '<:mate/d:' . $label,
+ %rel
+ );
+ }
- # Not unary
- elsif (!$_->{type}) {
+ # Not unary
+ elsif (!$_->{type}) {
- # Get information about the target token
- my $from = $_->{span}->{-from};
- my $to = $_->{span}->{-to};
+ # Get information about the target token
+ my $from = $_->{span}->{-from};
+ my $to = $_->{span}->{-to};
- # Get source node
- # This is only necessary for nodes with attributes
- # my $source_term = $stream->get_node(
- # $source, 'mate/d:' . $NODE_LABEL
- # );
+ # Get source node
+ # This is only necessary for nodes with attributes
+ # my $source_term = $stream->get_node(
+ # $source, 'mate/d:' . $NODE_LABEL
+ # );
- # Target
- my $target = $tokens->token($from, $to);
+ # Target
+ my $target = $tokens->token($from, $to);
- # Relation is term-to-term with a found target!
- if ($target) {
+ # Relation is term-to-term with a found target!
+ if ($target) {
- # Get target node
- # my $target_term = $stream->get_node(
- # $target, 'mate/d:' . $NODE_LABEL
- # );
+ # Get target node
+ # my $target_term = $stream->get_node(
+ # $target, 'mate/d:' . $NODE_LABEL
+ # );
- $mtt->add(
- term => '>:mate/d:' . $label,
- pti => 32, # term-to-term relation
- payload =>
- '<i>' . $target->pos # . # right part token position
-# '<s>0' . # $source_term->tui . # left part tui
-# '<s>0' # . $target_term->tui # right part tui
- );
+ $mtt->add(
+ term => '>:mate/d:' . $label,
+ pti => 32, # term-to-term relation
+ payload =>
+ '<i>' . $target->pos # . # right part token position
+ # '<s>0' . # $source_term->tui . # left part tui
+ # '<s>0' # . $target_term->tui # right part tui
+ );
- my $target_mtt = $stream->pos($target->pos);
- $target_mtt->add(
- term => '<:mate/d:' . $label,
- pti => 32, # term-to-term relation
- payload =>
- '<i>' . $source->pos # . # left part token position
-# '<s>0' . # $source_term->tui . # left part tui
-# '<s>0' # . $target_term->tui # right part tui
- );
- }
+ my $target_mtt = $stream->pos($target->pos);
+ $target_mtt->add(
+ term => '<:mate/d:' . $label,
+ pti => 32, # term-to-term relation
+ payload =>
+ '<i>' . $source->pos # . # left part token position
+ # '<s>0' . # $source_term->tui . # left part tui
+ # '<s>0' # . $target_term->tui # right part tui
+ );
+ }
- # Relation is possibly term-to-element with a found target!
- elsif ($target = $tokens->span($from, $to)) {
+ # Relation is possibly term-to-element with a found target!
+ elsif ($target = $tokens->span($from, $to)) {
- # Get target node
- # my $target_span = $stream->get_node(
- # $target, 'mate/d:' . $NODE_LABEL
- # );
+ # Get target node
+ # my $target_span = $stream->get_node(
+ # $target, 'mate/d:' . $NODE_LABEL
+ # );
- $mtt->add(
- term => '>:mate/d:' . $label,
- pti => 33, # term-to-element relation
- payload =>
- '<i>' . $target->o_start . # end position
- '<i>' . $target->o_end . # end position
- '<i>' . $target->p_start . # right part start position
- '<i>' . $target->p_end # . # right part end position
-# '<s>0' . # $source_term->tui . # left part tui
-# '<s>0' # . $target_span->tui # right part tui
- );
+ $mtt->add(
+ term => '>:mate/d:' . $label,
+ pti => 33, # term-to-element relation
+ payload =>
+ '<i>' . $target->o_start . # end position
+ '<i>' . $target->o_end . # end position
+ '<i>' . $target->p_start . # right part start position
+ '<i>' . $target->p_end # . # right part end position
+ # '<s>0' . # $source_term->tui . # left part tui
+ # '<s>0' # . $target_span->tui # right part tui
+ );
- my $target_mtt = $stream->pos($target->p_start);
- $target_mtt->add(
- term => '<:mate/d:' . $label,
- pti => 34, # element-to-term relation
- payload =>
- '<i>' . $target->o_start . # end position
- '<i>' . $target->o_end . # end position
- '<i>' . $target->p_end . # right part end position
- '<i>' . $source->pos # . # left part token position
-# '<s>0' . # $source_term->tui . # left part tui
-# '<s>0' # . $target_span->tui # right part tui
+ my $target_mtt = $stream->pos($target->p_start);
+ $target_mtt->add(
+ term => '<:mate/d:' . $label,
+ pti => 34, # element-to-term relation
+ payload =>
+ '<i>' . $target->o_start . # end position
+ '<i>' . $target->o_end . # end position
+ '<i>' . $target->p_end . # right part end position
+ '<i>' . $source->pos # . # left part token position
+ # '<s>0' . # $source_term->tui . # left part tui
+ # '<s>0' # . $target_span->tui # right part tui
- );
- }
- else {
- # use Data::Dumper;
- # warn '2###### ' . Dumper($content);
- };
- };
+ );
+ }
+ else {
+ # use Data::Dumper;
+ # warn '2###### ' . Dumper($content);
+ };
+ };
};
}) or return;
diff --git a/lib/KorAP/XML/Tokenizer/Range.pm b/lib/KorAP/XML/Tokenizer/Range.pm
index 366585c..8b3e992 100644
--- a/lib/KorAP/XML/Tokenizer/Range.pm
+++ b/lib/KorAP/XML/Tokenizer/Range.pm
@@ -80,18 +80,21 @@
return $2;
}
else {
- # I am not sure about that ...
- $found;
+ # The current position is likely wrong. The most common non-gap
+ # and non-exact-match is in the situation of
+ # "y<e> z</e>", where e is in the range of the y token.
+ $found + 1;
};
};
sub to_string {
my $self = shift;
- return join('',
- map {'['.join(',',@$_).']'}
- @{$$self->get_range(0,100,'...')}
- ) . '...';
+ return join(
+ '',
+ map {'['.join(',',@$_).']'}
+ @{$$self->get_range(0,100,'...')}
+ ) . '...';
};
1;
diff --git a/lib/KorAP/XML/Tokenizer/Span.pm b/lib/KorAP/XML/Tokenizer/Span.pm
index b359e14..7d7bd55 100644
--- a/lib/KorAP/XML/Tokenizer/Span.pm
+++ b/lib/KorAP/XML/Tokenizer/Span.pm
@@ -2,6 +2,7 @@
use strict;
use warnings;
use Mojo::DOM;
+use Clone;
sub new {
bless [], shift;
@@ -109,4 +110,11 @@
};
};
+
+# Clone the span
+sub clone {
+ return Clone::clone(shift);
+};
+
+
1;
diff --git a/lib/KorAP/XML/Tokenizer/Units.pm b/lib/KorAP/XML/Tokenizer/Units.pm
index 6b75e42..22f50d5 100644
--- a/lib/KorAP/XML/Tokenizer/Units.pm
+++ b/lib/KorAP/XML/Tokenizer/Units.pm
@@ -8,6 +8,8 @@
has 'have' => 0;
has 'encoding' => 'utf-8';
+use constant DEBUG => 0;
+
sub span {
my $self = shift;
my ($from, $to, $s) = @_;
@@ -19,11 +21,15 @@
$from //= 0;
# The span is invalid
- return unless $from <= $to;
+ unless ($from <= $to) {
+ if (DEBUG) {
+ warn $s->{-id} . ' is invalid';
+ };
+ return;
+ };
my $span = KorAP::XML::Tokenizer::Span->new;
-
# The span is a milestone
if ($from == $to) {
$span->milestone(1);
@@ -41,7 +47,13 @@
unless (defined $start) {
$start = $self->range->after($span->o_start);
- return unless defined $start;
+
+ unless (defined $start) {
+ if (DEBUG) {
+ warn $span->id . ' has no valid start';
+ };
+ return;
+ };
};
# Set start token position to span
@@ -57,25 +69,45 @@
unless (defined $end) {
$end = $self->range->before($span->o_end);
- return unless defined $end;
+
+ unless (defined $end) {
+ if (DEBUG) {
+ warn $span->id . ' has no valid end';
+ };
+ return;
+ };
# The next token of end has a character
- # offset AFTER th given end character offset
+ # offset AFTER the given end character offset
my $real_start = $self->stream->pos($end)->o_start;
# Ignore non-milestone elements outside the token stream!
- return if $to <= $real_start;
+ if ($to <= $real_start) {
+ if (DEBUG) {
+ warn 'Ignore ' . $span->id . ' is a non-milestone element outside the token stream';
+ };
+ return;
+ };
};
# $span->p_end($end);
# return unless $span->p_end >= $span->p_start;
# EXPERIMENTAL:
- return unless $end >= $span->p_start;
+ unless ($end >= $span->p_start) {
+ if (DEBUG) {
+ warn 'Ignore ' . $span->id . ' with ' . $span->p_start . '-' . $end;
+ };
+ return;
+ };
$span->p_end($end + 1);
}
+ if (DEBUG && $from == 124) {
+ warn 'exact: ' . $span->p_start . '-' . $span->p_end;
+ };
+
$span->hash($s) if $s;
$span;
@@ -116,6 +148,8 @@
$from = $p->bytes2chars($from);
$to = $p->bytes2chars($to);
}
+
+ # This is legacy treating of bytes2chars
elsif ($self->encoding eq 'xip') {
$from = $p->xip2chars($from);
$to = $p->xip2chars($to);
diff --git a/t/annotation/dereko_struct.t b/t/annotation/dereko_struct.t
index 5569369..4e00bb3 100644
--- a/t/annotation/dereko_struct.t
+++ b/t/annotation/dereko_struct.t
@@ -42,19 +42,19 @@
'@:dereko/s:pattern:text$<b>17<s>2',
'Attribute of idsHeader');
-is($data->{stream}->[4]->[1],
+is($data->{stream}->[5]->[1],
'<>:dereko/s:s$<b>64<i>32<i>42<i>6<b>6<s>1',
'Sentence span');
-is($data->{stream}->[4]->[2],
+is($data->{stream}->[5]->[2],
'@:dereko/s:broken:no$<b>17<s>1<i>6',
'Attribute of sentence span');
-is($data->{stream}->[6]->[0],
- '<>:dereko/s:pb$<b>65<i>42<i>42<i>6<b>6<s>1',
+is($data->{stream}->[7]->[0],
+ '<>:dereko/s:pb$<b>65<i>42<i>42<i>7<b>6<s>1',
'Pagebreak element');
-is($data->{stream}->[6]->[-1],
+is($data->{stream}->[7]->[-1],
'~:base/s:pb$<i>2<i>42',
'Pagebreak element');
diff --git a/t/corpus/REDEW/DOC1/00000/data.xml b/t/corpus/REDEW/DOC1/00000/data.xml
new file mode 100644
index 0000000..3b85204
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/data.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="text.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<raw_text docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP">
+ <metadata file="metadata.xml" />
+ <text>Cechov,_Anton_Pavlovic_Gram.tg4_1.xml 1886 1880 Gram Čechov, Anton Pavlovič yes yes Erzähltext digbib no third chevron ... Hörst du, du alter Drachen? Oder machst du dir nichts aus unseren Worten?«</text>
+</raw_text>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml b/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml
new file mode 100644
index 0000000..66f7ba1
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml
@@ -0,0 +1,205 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+ <spanList>
+ <span id="s0" from="119" to="122" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">--</f>
+ <f name="pos">$(</f>
+ <f name="msd">msd=rfpos=SYM.Pun.Cont|sentstart=yes|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s1" from="123" to="128" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">Hörst</f>
+ <f name="pos">VVFIN</f>
+ <f name="msd">msd=rfpos=VFIN.Full.2.Sg.Pres.Ind|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s2" from="129" to="131" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">du</f>
+ <f name="pos">PPER</f>
+ <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s3" from="131" to="132" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">--</f>
+ <f name="pos">$,</f>
+ <f name="msd">msd=rfpos=SYM.Pun.Comma|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s4" from="133" to="135" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">du</f>
+ <f name="pos">PPER</f>
+ <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s5" from="136" to="141" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">alt</f>
+ <f name="pos">ADJA</f>
+ <f name="msd">msd=rfpos=ADJA.Pos.Gen.Pl.Masc|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s6" from="142" to="149" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">Drache</f>
+ <f name="pos">NN</f>
+ <f name="msd">msd=rfpos=N.Reg.Gen.Pl.Masc|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s7" from="149" to="150" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">--</f>
+ <f name="pos">$.</f>
+ <f name="msd">msd=rfpos=SYM.Pun.Sent|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s8" from="151" to="155" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">oder</f>
+ <f name="pos">NE</f>
+ <f name="msd">msd=rfpos=CONJ.Coord.-|sentstart=yes|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s9" from="156" to="162" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">machen</f>
+ <f name="pos">VVFIN</f>
+ <f name="msd">msd=rfpos=VFIN.Full.2.Sg.Pres.Ind|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s10" from="163" to="165" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">du</f>
+ <f name="pos">PPER</f>
+ <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s11" from="166" to="169" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">dir</f>
+ <f name="pos">PPER</f>
+ <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Dat.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s12" from="170" to="176" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">nichts</f>
+ <f name="pos">PIS</f>
+ <f name="msd">msd=rfpos=PRO.Indef.Subst.-.*.*.Neut|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s13" from="177" to="180" l="6">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">aus</f>
+ <f name="pos">APPR</f>
+ <f name="msd">msd=rfpos=APPR.Dat|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s14" from="181" to="188" l="7">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">unser</f>
+ <f name="pos">PPOSAT</f>
+ <f name="msd">msd=rfpos=PRO.Poss.Attr.-.Dat.Pl.Neut|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s15" from="189" to="195" l="7">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">Wort</f>
+ <f name="pos">NN</f>
+ <f name="msd">msd=rfpos=N.Reg.Dat.Pl.Neut|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s16" from="195" to="196" l="6">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">--</f>
+ <f name="pos">$.</f>
+ <f name="msd">msd=rfpos=SYM.Pun.Sent|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s17" from="196" to="197" l="5">
+ <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="lex">
+ <fs>
+ <f name="lemma">"</f>
+ <f name="pos">$(</f>
+ <f name="msd">msd=rfpos=N.Reg.Nom.Sg.Masc|sentstart=no|stwr=direct.speech.1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/header.xml b/t/corpus/REDEW/DOC1/00000/header.xml
new file mode 100644
index 0000000..9738cd3
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/header.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="text" pattern="text" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+ <title>Gram</title>
+<!--- missing in origin data -->
+<textSigle>REDEW/DOC1.00000</textSigle>
+<!-- -->
+ <author>Čechov, Anton Pavlovič</author>
+ <respStmt>
+ <resp></resp>
+ <name></name>
+ </respStmt>
+ </titleStmt>
+ <publicationStmt>
+ <distributor>www.redewiedergabe.de</distributor>
+ </publicationStmt>
+ <sourceDesc>
+ <bibl></bibl>
+ </sourceDesc>
+ </fileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/struct/structure.xml b/t/corpus/REDEW/DOC1/00000/struct/structure.xml
new file mode 100644
index 0000000..c12f102
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/struct/structure.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+ <spanList>
+ <span id="s0" from="0" to="197" l="1">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">text</f>
+ </fs>
+ </span>
+ <span id="s1" from="0" to="118" l="2">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">fs</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="type">metadata</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s2" from="0" to="37" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">filename</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s3" from="38" to="42" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">year</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s4" from="43" to="47" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">decade</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s5" from="48" to="52" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">title</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s6" from="53" to="75" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">author</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s7" from="76" to="79" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">fictional</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s8" from="80" to="83" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">narrative</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s9" from="84" to="94" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">textype</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s10" from="95" to="101" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">source</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s11" from="102" to="104" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">dialect</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s12" from="105" to="110" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">perspective</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s13" from="111" to="118" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">f</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="name">quotes</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s14" from="118" to="197" l="2">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">body</f>
+ </fs>
+ </span>
+ <span id="s15" from="118" to="197" l="3">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">p</f>
+ </fs>
+ </span>
+ <span id="s16" from="118" to="197" l="4">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">said</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="id">1</f>
+ <f name="mode">direct</f>
+ <f name="content">speech</f>
+ <f name="level">1</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s30" from="176" to="196" l="5">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">said</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="id">19</f>
+ <f name="mode">reported</f>
+ <f name="content">speech</f>
+ <f name="level">2</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s32" from="180" to="188" l="6">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">seg</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="id">19</f>
+ <f name="type">speaker</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ <span id="s34" from="188" to="195" l="6">
+ <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+ <f name="name">seg</f>
+ <f name="attr">
+ <fs type="attr">
+ <f name="id">19</f>
+ <f name="type">intExpr</f>
+ </fs>
+ </f>
+ </fs>
+ </span>
+ </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/header.xml b/t/corpus/REDEW/DOC1/header.xml
new file mode 100644
index 0000000..b64c4db
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/header.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="document" pattern="text" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+<!--- missing in origin data -->
+ <dokumentSigle>REDEW/DOC1</dokumentSigle>
+<!-- -->
+ <d.title>Redewiedergabe Dokument 1</d.title>
+ </titleStmt>
+ <publicationStmt>
+ <distributor/>
+ <pubAddress/>
+ <availability region="world">QAO-NC</availability>
+ <pubDate/>
+ </publicationStmt>
+ <sourceDesc>
+ <biblStruct>
+ <monogr>
+ <h.title type="main"/>
+ <edition>
+ <further/>
+ <kind/>
+ <appearance/>
+ </edition>
+ <imprint/>
+ <biblNote n="1">?</biblNote>
+ </monogr>
+ </biblStruct>
+ </sourceDesc>
+ </fileDesc>
+ <profileDesc>
+ <creation>
+ <creatDate>2020</creatDate>
+ <creatRef>?</creatRef>
+ <creatRefShort>?</creatRefShort>
+ </creation>
+ <textDesc>
+ <textType>?</textType>
+ <textTypeRef/>
+ </textDesc>
+ </profileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/corpus/REDEW/header.xml b/t/corpus/REDEW/header.xml
new file mode 100644
index 0000000..dc35127
--- /dev/null
+++ b/t/corpus/REDEW/header.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="corpus" pattern="allesaußerZtg/Zschr" status="new" version="1.1" TEIform="teiHeader">
+ <fileDesc>
+ <titleStmt>
+<!--- missing in origin data -->
+ <korpusSigle>REDEW</korpusSigle>
+<!-- -->
+ <c.title>Redewiedergabe</c.title>
+ </titleStmt>
+ <publicationStmt>
+ <distributor> Institut für Deutsche Sprache </distributor>
+ <pubAddress> Postfach 10 16 21, D-68016 Mannheim </pubAddress>
+ <telephone> +49 (0)621 1581 0 </telephone>
+ <availability region="world">QAO-NC</availability>
+ <pubDate/>
+ </publicationStmt>
+ <sourceDesc>
+ <biblFull>
+ <titleStmt>
+ <x.title/>
+ </titleStmt>
+ <editionStmt/>
+ <publicationStmt>
+ <distributor/>
+ <pubAddress/>
+ <availability region="world">QAO-NC</availability>
+ <pubDate/>
+ </publicationStmt>
+ </biblFull>
+ <biblStruct>
+ <monogr>
+ <h.title type="main">Redewiedergabe</h.title>
+ <h.title type="sub"/>
+ <h.author>...</h.author>
+ <editor>...</editor>
+ <edition>
+ <further/>
+ <kind></kind>
+ <appearance/>
+ </edition>
+ <imprint>
+ <publisher>...</publisher>
+ <pubDate type="year">2020</pubDate>
+ <pubDate type="month"/>
+ <pubDate type="day"/>
+ <pubPlace key="DE"></pubPlace>
+ </imprint>
+ </monogr>
+ </biblStruct>
+ <reference type="super" assemblage="non-automatic">...</reference>
+ </sourceDesc>
+ </fileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/range.t b/t/range.t
index 27c7970..2091661 100644
--- a/t/range.t
+++ b/t/range.t
@@ -44,8 +44,8 @@
is($range->after(0), 0, 'After is fine');
is($range->after(1), 0, 'After is fine');
is($range->after(2), 0, 'After is fine');
-is($range->after(3), 0, 'After is fine');
-is($range->after(14), 0, 'After is fine');
+is($range->after(3), 1, 'After is fine');
+is($range->after(14), 1, 'After is fine');
is($range->after(15), 1, 'After is fine');
is($range->to_string,
diff --git a/t/real/agd.t b/t/real/agd.t
index 6ba7ebe..7745bea 100644
--- a/t/real/agd.t
+++ b/t/real/agd.t
@@ -118,12 +118,12 @@
$first_token = join('||', @{$output->{data}->{stream}->[0]});
like($first_token, qr!<>:base/s:s\$<b>64<i>0<i>16<i>2<b>1!);
-my $token = join('||', @{$output->{data}->{stream}->[1]});
+my $token = join('||', @{$output->{data}->{stream}->[2]});
like($token, qr!<>:base/s:s\$<b>64<i>16<i>23<i>4<b>1!);
-$token = join('||', @{$output->{data}->{stream}->[2]});
+$token = join('||', @{$output->{data}->{stream}->[3]});
unlike($token, qr!<>:base/s:s!);
-$token = join('||', @{$output->{data}->{stream}->[3]});
+$token = join('||', @{$output->{data}->{stream}->[4]});
like($token, qr!<>:base/s:s\$<b>64<i>23<i>27<i>5<b>1!);
$token = join('||', @{$output->{data}->{stream}->[5]});
diff --git a/t/real/redew.t b/t/real/redew.t
new file mode 100644
index 0000000..25f369c
--- /dev/null
+++ b/t/real/redew.t
@@ -0,0 +1,155 @@
+use strict;
+use warnings;
+use Test::More;
+use Data::Dumper;
+use JSON::XS;
+use Log::Log4perl;
+
+use utf8;
+use lib 'lib', '../lib';
+
+use File::Basename 'dirname';
+use File::Spec::Functions 'catdir';
+
+# Initialize log4perl object
+Log::Log4perl->init({
+ 'log4perl.rootLogger' => 'ERROR, STDERR',
+ 'log4perl.appender.STDERR' => 'Log::Log4perl::Appender::ScreenColoredLevels',
+ 'log4perl.appender.STDERR.layout' => 'PatternLayout',
+ 'log4perl.appender.STDERR.layout.ConversionPattern' => '[%r] %F %L %c - %m%n'
+});
+
+use_ok('KorAP::XML::Krill');
+
+my $path = catdir(dirname(__FILE__), '../corpus/REDEW/DOC1/00000');
+
+ok(my $doc = KorAP::XML::Krill->new( path => $path . '/' ), 'Load Korap::Document');
+ok($doc->parse, 'Parse document');
+
+is($doc->text_sigle, 'REDEW/DOC1/00000', 'Correct text sigle');
+is($doc->doc_sigle, 'REDEW/DOC1', 'Correct document sigle');
+is($doc->corpus_sigle, 'REDEW', 'Correct corpus sigle');
+
+my $meta = $doc->meta;
+ok(!$meta->{T_title}, 'Title'); # ???
+ok(!$meta->{T_sub_title}, 'SubTitle');
+ok(!$meta->{T_author}, 'Author');
+ok(!$meta->{A_editor}, 'Editor');
+ok(!$meta->{S_pub_place}, 'PubPlace');
+is($meta->{A_publisher}, '...', 'Publisher'); # ???
+
+is($meta->{S_text_type}, '?', 'Text Type'); # ???
+ok(!$meta->{S_text_type_art}, 'No Text Type Art');
+ok(!$meta->{S_text_type_ref}, 'No Text Type Ref');
+ok(!$meta->{S_text_domain}, 'No Text Domain');
+ok(!$meta->{S_text_column}, 'No Text Column');
+
+ok(!$meta->{K_text_class}->[0], 'Correct Text Class');
+
+is($meta->{D_pub_date}, '20200000', 'Creation date');
+is($meta->{D_creation_date}, '20200000', 'Creation date');
+is($meta->{S_availability}, 'QAO-NC', 'License'); # ???
+ok(!$meta->{A_pages}, 'Pages');
+
+ok(!$meta->{A_file_edition_statement}, 'File Statement');
+ok(!$meta->{A_bibl_edition_statement}, 'Bibl Statement');
+
+ok(!$meta->{A_reference}, 'Reference');
+ok(!$meta->{S_language}, 'Language'); # ???
+
+is($meta->{T_corpus_title}, 'Redewiedergabe', 'Correct Corpus title');
+ok(!$meta->{T_corpus_sub_title}, 'Correct Corpus sub title');
+is($meta->{T_corpus_author}, '...', 'Correct Corpus author'); # ???
+is($meta->{A_corpus_editor}, '...', 'Correct Corpus editor'); # ???
+
+is($meta->{T_doc_title}, 'Redewiedergabe Dokument 1', 'Correct Doc title');
+ok(!$meta->{T_doc_sub_title}, 'Correct Doc sub title');
+ok(!$meta->{T_doc_author}, 'Correct Doc author');
+ok(!$meta->{A_doc_editor}, 'Correct doc editor');
+
+# Tokenization
+use_ok('KorAP::XML::Tokenizer');
+
+my ($token_base_foundry, $token_base_layer) = (qw/drukola Morpho/);
+
+# Get tokenization
+my $tokens = KorAP::XML::Tokenizer->new(
+ path => $doc->path,
+ doc => $doc,
+ foundry => $token_base_foundry,
+ layer => $token_base_layer,
+ name => 'tokens'
+);
+
+ok($tokens, 'Token Object is fine');
+ok($tokens->parse, 'Token parsing is fine');
+
+my $output = decode_json( $tokens->to_json );
+
+is(substr($output->{data}->{text}, 0, 100), 'Cechov,_Anton_Pavlovic_Gram.tg4_1.xml 1886 1880 Gram Čechov, Anton Pavlovič yes yes Erzähltext digbi', 'Primary Data');
+
+is($output->{data}->{name}, 'tokens', 'tokenName');
+is($output->{data}->{tokenSource}, 'drukola#morpho', 'tokenSource');
+is($output->{version}, '0.03', 'version');
+
+is($output->{data}->{foundries}, '', 'Foundries');
+is($output->{data}->{layerInfos}, '', 'layerInfos');
+is($output->{data}->{stream}->[0]->[4], 's:Hörst', 'data');
+
+is($output->{textSigle}, 'REDEW/DOC1/00000', 'Correct text sigle');
+is($output->{docSigle}, 'REDEW/DOC1', 'Correct document sigle');
+is($output->{corpusSigle}, 'REDEW', 'Correct corpus sigle');
+
+ok(!$output->{title}, 'Title');
+ok(!$output->{subTitle}, 'Correct SubTitle');
+ok(!$output->{author}, 'Author');
+ok(!exists $output->{editor}, 'Publisher');
+
+# Add annotations
+$tokens->add('DRuKoLa', 'Morpho');
+$tokens->add('DeReKo', 'Structure');
+
+$output = decode_json( $tokens->to_json );
+
+my $first = $output->{data}->{stream}->[0];
+
+is('-:tokens$<i>13',$first->[0]);
+is('<>:base/s:t$<b>64<i>0<i>197<i>12<b>0',$first->[1]);
+is('<>:dereko/s:text$<b>64<i>0<i>197<i>12<b>0',$first->[2]);
+is('<>:dereko/s:body$<b>64<i>118<i>197<i>12<b>1',$first->[3]);
+is('<>:dereko/s:p$<b>64<i>118<i>197<i>12<b>2',$first->[4]);
+is('<>:dereko/s:said$<b>64<i>118<i>197<i>12<b>3<s>1',$first->[5]);
+is('@:dereko/s:level:1$<b>17<s>1<i>12',$first->[6]);
+is('@:dereko/s:content:speech$<b>17<s>1<i>12',$first->[7]);
+is('@:dereko/s:mode:direct$<b>17<s>1<i>12',$first->[8]);
+is('@:dereko/s:id:1$<b>17<s>1<i>12',$first->[9]);
+is('_0$<i>123<i>128',$first->[10]);
+is("drukola/l:H\x{f6}rst",$first->[11]);
+is('drukola/m:msd:rfpos',$first->[12]);
+is('drukola/m:sentstart:no',$first->[13]);
+is('drukola/m:stwr:direct.speech.1',$first->[14]);
+is('drukola/p:VVFIN',$first->[15]);
+is("i:h\x{f6}rst",$first->[16]);
+is("s:H\x{f6}rst",$first->[17]);
+
+my $nine = join(',', @{$output->{data}->{stream}->[9]});
+like($nine, qr{drukola\/l:nichts}, 'Nichts');
+like($nine, qr{_9\$<i>170<i>176}, 'Term boundaries');
+unlike($nine, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>12<b>4<s>1}, 'Term boundaries');
+
+my $ten = join(',', @{$output->{data}->{stream}->[10]});
+like($ten, qr{_10\$<i>177<i>180}, 'Term boundaries');
+like($ten, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>12<b>4<s>1}, 'Term boundaries');
+
+my $eleven = join(',', @{$output->{data}->{stream}->[11]});
+like($eleven, qr{_11\$<i>181<i>188}, 'Term boundaries');
+like($eleven, qr{<>:dereko/s:seg\$<b>64<i>180<i>188<i>12<b>5<s>1}, 'Segment');
+
+
+my $twelve = join(',', @{$output->{data}->{stream}->[12]});
+like($twelve, qr{_12\$<i>189<i>195}, 'Term boundaries');
+like($twelve, qr{drukola/l:Wort}, 'Lemma');
+like($twelve, qr{<>:dereko/s:seg\$<b>64<i>188<i>195<i>13<b>5<s>1}, 'Segment');
+
+done_testing;
+__END__
diff --git a/t/real/rei.t b/t/real/rei.t
index b375c24..d50a732 100644
--- a/t/real/rei.t
+++ b/t/real/rei.t
@@ -225,16 +225,17 @@
my $last = $output->{data}->{stream}->[-1];
-is('<>:dereko/s:text$<b>65<i>17859<i>17859<i>2640<b>0', $last->[0]);
-is('<>:dereko/s:back$<b>65<i>17859<i>17859<i>2640<b>1', $last->[1]);
-is('>:malt/d:APP$<b>32<i>2639', $last->[2]);
-is('_2640$<i>17851<i>17859', $last->[3]);
-is('corenlp/p:NE', $last->[4]);
-is("i:schr\x{f6}der", $last->[5]);
-is('opennlp/p:NE', $last->[6]);
-is("s:Schr\x{f6}der", $last->[7]);
-is("tt/l:Schr\x{f6}der", $last->[8]);
-is('tt/p:NE', $last->[9]);
+# Milestones behind the final token are no longer indexed
+# is('<>:dereko/s:text$<b>65<i>17859<i>17859<i>2640<b>0', $last->[0]);
+# is('<>:dereko/s:back$<b>65<i>17859<i>17859<i>2640<b>1', $last->[1]);
+is('>:malt/d:APP$<b>32<i>2639', $last->[0]);
+is('_2640$<i>17851<i>17859', $last->[1]);
+is('corenlp/p:NE', $last->[2]);
+is("i:schr\x{f6}der", $last->[3]);
+is('opennlp/p:NE', $last->[4]);
+is("s:Schr\x{f6}der", $last->[5]);
+is("tt/l:Schr\x{f6}der", $last->[6]);
+is('tt/p:NE', $last->[7]);
# REI/BNG/00071
@@ -282,7 +283,7 @@
is('-:corenlp/sentences$<i>69', $first->[2]);
is('-:opennlp/sentences$<i>65', $first->[3]);
is('-:tokens$<i>1009', $first->[4]);
-is('-:tt/sentences$<i>95', $first->[5]);
+is('-:tt/sentences$<i>94', $first->[5]);
is('<:malt/d:APP$<b>32<i>1', $first->[6]);
is('<>:corenlp/c:MPN$<b>64<i>0<i>16<i>2<b>3', $first->[7]);
is('<>:base/s:s$<b>64<i>0<i>48<i>5<b>2', $first->[8]);
diff --git a/t/real/wpd.t b/t/real/wpd.t
index 517f9be..e45db0a 100644
--- a/t/real/wpd.t
+++ b/t/real/wpd.t
@@ -78,10 +78,17 @@
my $stream = $tokens->to_data->{data}->{stream};
# This is not a goot relation example
-is($stream->[77]->[0],
- '<:mate/d:--$<b>34<i>498<i>499<i>78<i>78',
- 'element to term');
-is($stream->[78]->[0], '>:mate/d:--$<b>33<i>498<i>499<i>77<i>78', 'term to element');
+ is($stream->[79]->[0],
+ '>:mate/d:CJ$<b>32<i>68',
+ 'term to term');
+ is($stream->[79]->[1], '<:mate/d:PD$<b>32<i>81', 'term to term');
+
+
+# These are no longer aligned
+# is($stream->[77]->[0],
+# '<:mate/d:--$<b>34<i>498<i>499<i>78<i>78',
+# 'element to term');
+# is($stream->[78]->[0], '>:mate/d:--$<b>33<i>498<i>499<i>77<i>78', 'term to element');
$tokens->add('Base', 'Sentences');
diff --git a/t/real/wpf.t b/t/real/wpf.t
index 4a28c2f..ba6fb94 100644
--- a/t/real/wpf.t
+++ b/t/real/wpf.t
@@ -101,10 +101,11 @@
$tokens->add('Talismane', 'Morpho');
$stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[10], 'talismane/l:Sigmund', 'Talismane morpho annotation');
-is($stream->[1]->[11], 'talismane/m:g:m', 'Talismane morpho annotation');
-is($stream->[1]->[12], 'talismane/m:n:s', 'Talismane morpho annotation');
-is($stream->[1]->[13], 'talismane/p:NPP', 'Talismane morpho annotation');
+
+is($stream->[1]->[9], 'talismane/l:Sigmund', 'Talismane morpho annotation');
+is($stream->[1]->[10], 'talismane/m:g:m', 'Talismane morpho annotation');
+is($stream->[1]->[11], 'talismane/m:n:s', 'Talismane morpho annotation');
+is($stream->[1]->[12], 'talismane/p:NPP', 'Talismane morpho annotation');
is($stream->[300]->[5], 'talismane/l:son', 'Talismane lemma annotation');
is($stream->[300]->[6], 'talismane/m:g:m', 'Talismane morph annotation');
@@ -116,7 +117,10 @@
# Add Malt dependency
$tokens->add('Malt', 'Dependency');
$stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[1], '>:malt/d:dep$<b>33<i>7<i>8<i>0<i>1', 'Malt dep annotation');
+
+# This is no longer indexed
+# is($stream->[1]->[1], '>:malt/d:dep$<b>33<i>7<i>8<i>0<i>1', 'Malt dep annotation');
+
is($stream->[1]->[2], '<:malt/d:dep$<b>32<i>2', 'Malt dep annotation');
is($stream->[300]->[1], '>:malt/d:punct$<b>32<i>302', 'Malt dep annotation');
@@ -124,9 +128,10 @@
# Add TreeTagger morpho
$tokens->add('TreeTagger', 'Morpho');
$stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[14], 'tt/p:ADJ$<b>129<b>26', 'TreeTagger morph annotation');
-is($stream->[1]->[15], 'tt/p:NAM$<b>129<b>200', 'TreeTagger morph annotation');
-is($stream->[1]->[16], 'tt/p:NOM$<b>129<b>27', 'TreeTagger morph annotation');
+
+is($stream->[1]->[13], 'tt/p:ADJ$<b>129<b>26', 'TreeTagger morph annotation');
+is($stream->[1]->[14], 'tt/p:NAM$<b>129<b>200', 'TreeTagger morph annotation');
+is($stream->[1]->[15], 'tt/p:NOM$<b>129<b>27', 'TreeTagger morph annotation');
done_testing;
diff --git a/t/script/non_verbal_tokens.t b/t/script/non_verbal_tokens.t
index fa460e4..5327523 100644
--- a/t/script/non_verbal_tokens.t
+++ b/t/script/non_verbal_tokens.t
@@ -54,7 +54,7 @@
my $token = $stream->[4];
is($token->[3], 'dgd/l:pui', 'Token');
$token = $stream->[5];
-is($token->[13], 'dgd/l:xui', 'Token');
+is($token->[15], 'dgd/l:xui', 'Token');
$call = join(
' ',
@@ -88,10 +88,10 @@
is($token->[3], 'dgd/l:pui', 'Token');
$token = $stream->[5];
-is($token->[14], 'dgd/para:pause$<b>128<s>5', 'Token');
+is($token->[4], 'dgd/para:pause$<b>128<s>2', 'Token');
$token = $stream->[6];
-is($token->[1], 'dgd/l:xui', 'Token');
+is($token->[13], 'dgd/l:xui', 'Token');
diff --git a/t/script/non_word_tokens.t b/t/script/non_word_tokens.t
index 0eb9e14..b156866 100644
--- a/t/script/non_word_tokens.t
+++ b/t/script/non_word_tokens.t
@@ -51,7 +51,7 @@
is($json->{data}->{foundries}, 'base base/paragraphs base/sentences connexor connexor/morpho connexor/phrase connexor/sentences connexor/syntax corenlp corenlp/sentences dereko dereko/structure mate mate/dependency mate/morpho opennlp opennlp/morpho opennlp/sentences treetagger treetagger/morpho treetagger/sentences xip xip/constituency xip/morpho xip/sentences', 'Foundries');
my $stream = $json->{data}->{stream};
my $token = $stream->[12];
-is($token->[17], 's:Vokal', 'Token');
+is($token->[16], 's:Vokal', 'Token');
$token = $stream->[13];
is($token->[23], 's:Der', 'Token');
diff --git a/t/transform.t b/t/transform.t
index 4537bcb..97c5a8b 100644
--- a/t/transform.t
+++ b/t/transform.t
@@ -154,7 +154,7 @@
'<>:opennlp/s:s$<b>64<i>0<i>74<i>13<b>0|'.
'-:opennlp/sentences$<i>50|'.
'<>:corenlp/s:s$<b>64<i>0<i>6<i>2<b>0|'.
- '-:corenlp/sentences$<i>68|'.
+ '-:corenlp/sentences$<i>67|'.
'cnx/l:A|'.
'cnx/p:N|'.
'cnx/syn:@NH|'.