Fixed span position offset bug and fixed milestones behind last token bug

Change-Id: I6250d515ec132e9024b1eaca82523887b7d6f643
diff --git a/Changes b/Changes
index 053e517..a351a92 100644
--- a/Changes
+++ b/Changes
@@ -1,5 +1,10 @@
 0.40 2020-03-01
         - Fixed XIP parser.
+        - Added example corpus of the
+          Redewiedergabe-Korpus.
+        - Fixed span offset bug.
+        - Fixed milestones behind the last
+          token bug.
 
 0.39 2020-02-19
         - Added Talismane support.
diff --git a/lib/KorAP/XML/Annotation/DGD/Structure.pm b/lib/KorAP/XML/Annotation/DGD/Structure.pm
index 6961b45..347b6c0 100644
--- a/lib/KorAP/XML/Annotation/DGD/Structure.pm
+++ b/lib/KorAP/XML/Annotation/DGD/Structure.pm
@@ -73,7 +73,7 @@
       o_start => $o_start,
       o_end   => $_->[1],
       p_start => $p_start,
-      p_end   => $_->[0] + 1,
+      p_end   => $_->[0],
       pti     => 64
     );
     $mt->payload('<b>1');
diff --git a/lib/KorAP/XML/Annotation/DeReKo/Structure.pm b/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
index 769e629..cc2b751 100644
--- a/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
+++ b/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
@@ -36,6 +36,16 @@
       # Get the mtt
       my $mtt = $stream->pos($p_start);
 
+      unless ($mtt) {
+
+        # This is a special case were a milestone is at the
+        # end of a text and can't be indexed at the moment!
+        $$self->log->warn(
+          'Span ' . $span->to_string . ' can\'t be indexed'
+        );
+        return;
+      };
+
       my $p_end = $span->p_end;
 
       # Add structure
diff --git a/lib/KorAP/XML/Annotation/Mate/Dependency.pm b/lib/KorAP/XML/Annotation/Mate/Dependency.pm
index ffc3223..6d242c9 100644
--- a/lib/KorAP/XML/Annotation/Mate/Dependency.pm
+++ b/lib/KorAP/XML/Annotation/Mate/Dependency.pm
@@ -30,126 +30,126 @@
 
       # Iterate over relations
       foreach (@$rel) {
-	my $label = $_->{-label};
+        my $label = $_->{-label};
 
-	# Relation type is unary
-	# Unary means, it refers to itself!
-	if ($_->{-type} && $_->{-type} eq 'unary') {
+        # Relation type is unary
+        # Unary means, it refers to itself!
+        if ($_->{-type} && $_->{-type} eq 'unary') {
 
-	  # I have no clue, what -- should mean
-	  # next if $_->{-label} eq '--';
+          # I have no clue, what -- should mean
+          # next if $_->{-label} eq '--';
 
-	  # Get target node - not very elegant
-	  # This is only necessary for nodes with attributes
-	  #	  my $target = $stream->get_node(
-	  #	    $source, 'mate/d:' . $NODE_LABEL
-	  #	  );
+          # Get target node - not very elegant
+          # This is only necessary for nodes with attributes
+          #    my $target = $stream->get_node(
+          #      $source, 'mate/d:' . $NODE_LABEL
+          #    );
 
-	  # Target is at the same position!
-	  my $pos = $source->pos;
+          # Target is at the same position!
+          my $pos = $source->pos;
 
-	  my %rel = (
-	    pti => 32, # term-to-term relation
-	    payload =>
-	      '<i>' . $pos # . # right part token position
-#		'<s>0' . # $target->tui . # left part tui
-#		  '<s>0'  # . $target->tui # right part tui
-		);
+          my %rel = (
+            pti => 32, # term-to-term relation
+            payload =>
+              '<i>' . $pos # . # right part token position
+              #    '<s>0' . # $target->tui . # left part tui
+              #      '<s>0'  # . $target->tui # right part tui
+            );
 
-	  # Add relations
-	  $mtt->add(
-	    term => '>:mate/d:' . $label,
-	    %rel
-	  );
-	  $mtt->add(
-	    term => '<:mate/d:' . $label,
-	    %rel
-	  );
-	}
+          # Add relations
+          $mtt->add(
+            term => '>:mate/d:' . $label,
+            %rel
+          );
+          $mtt->add(
+            term => '<:mate/d:' . $label,
+            %rel
+          );
+        }
 
-	# Not unary
-	elsif (!$_->{type}) {
+        # Not unary
+        elsif (!$_->{type}) {
 
-	  # Get information about the target token
-	  my $from = $_->{span}->{-from};
-	  my $to   = $_->{span}->{-to};
+          # Get information about the target token
+          my $from = $_->{span}->{-from};
+          my $to   = $_->{span}->{-to};
 
-	  # Get source node
-	  # This is only necessary for nodes with attributes
-	  #	  my $source_term = $stream->get_node(
-	  #	    $source, 'mate/d:' . $NODE_LABEL
-	  #	  );
+          # Get source node
+          # This is only necessary for nodes with attributes
+          #    my $source_term = $stream->get_node(
+          #      $source, 'mate/d:' . $NODE_LABEL
+          #    );
 
-	  # Target
-	  my $target = $tokens->token($from, $to);
+          # Target
+          my $target = $tokens->token($from, $to);
 
-	  # Relation is term-to-term with a found target!
-	  if ($target) {
+          # Relation is term-to-term with a found target!
+          if ($target) {
 
-	    # Get target node
-	    #	    my $target_term = $stream->get_node(
-	    #	      $target, 'mate/d:' . $NODE_LABEL
-	    #	    );
+            # Get target node
+            #      my $target_term = $stream->get_node(
+            #        $target, 'mate/d:' . $NODE_LABEL
+            #      );
 
-	    $mtt->add(
-	      term => '>:mate/d:' . $label,
-	      pti => 32, # term-to-term relation
-	      payload =>
-		'<i>' . $target->pos # . # right part token position
-#		  '<s>0' . # $source_term->tui . # left part tui
-#		    '<s>0' # . $target_term->tui # right part tui
-	    );
+            $mtt->add(
+              term => '>:mate/d:' . $label,
+              pti => 32, # term-to-term relation
+              payload =>
+                '<i>' . $target->pos # . # right part token position
+                #      '<s>0' . # $source_term->tui . # left part tui
+                #        '<s>0' # . $target_term->tui # right part tui
+              );
 
-	    my $target_mtt = $stream->pos($target->pos);
-	    $target_mtt->add(
-	      term => '<:mate/d:' . $label,
-	      pti => 32, # term-to-term relation
-	      payload =>
-		'<i>' . $source->pos # . # left part token position
-#		  '<s>0' . # $source_term->tui . # left part tui
-#		    '<s>0' # . $target_term->tui # right part tui
-	    );
-	  }
+            my $target_mtt = $stream->pos($target->pos);
+            $target_mtt->add(
+              term => '<:mate/d:' . $label,
+              pti => 32, # term-to-term relation
+              payload =>
+                '<i>' . $source->pos # . # left part token position
+                #      '<s>0' . # $source_term->tui . # left part tui
+                #        '<s>0' # . $target_term->tui # right part tui
+              );
+          }
 
-	  # Relation is possibly term-to-element with a found target!
-	  elsif ($target = $tokens->span($from, $to)) {
+          # Relation is possibly term-to-element with a found target!
+          elsif ($target = $tokens->span($from, $to)) {
 
-	    # Get target node
-	    #	    my $target_span = $stream->get_node(
-	    #	      $target, 'mate/d:' . $NODE_LABEL
-	    #	    );
+            # Get target node
+            #      my $target_span = $stream->get_node(
+            #        $target, 'mate/d:' . $NODE_LABEL
+            #      );
 
-	    $mtt->add(
-	      term => '>:mate/d:' . $label,
-	      pti => 33, # term-to-element relation
-	      payload =>
-		'<i>' . $target->o_start . # end position
-		  '<i>' . $target->o_end . # end position
-		    '<i>' . $target->p_start . # right part start position
-		      '<i>' . $target->p_end # . # right part end position
-#			'<s>0' . # $source_term->tui . # left part tui
-#			  '<s>0' # . $target_span->tui # right part tui
-	    );
+            $mtt->add(
+              term => '>:mate/d:' . $label,
+              pti => 33, # term-to-element relation
+              payload =>
+                '<i>' . $target->o_start . # end position
+                '<i>' . $target->o_end . # end position
+                '<i>' . $target->p_start . # right part start position
+                '<i>' . $target->p_end # . # right part end position
+                #      '<s>0' . # $source_term->tui . # left part tui
+                #        '<s>0' # . $target_span->tui # right part tui
+              );
 
-	    my $target_mtt = $stream->pos($target->p_start);
-	    $target_mtt->add(
-	      term => '<:mate/d:' . $label,
-	      pti => 34, # element-to-term relation
-	      payload =>
-		'<i>' . $target->o_start . # end position
-		  '<i>' . $target->o_end . # end position
-		    '<i>' . $target->p_end . # right part end position
-		      '<i>' . $source->pos # . # left part token position
-#			'<s>0' . # $source_term->tui . # left part tui
-#			  '<s>0' # . $target_span->tui # right part tui
+            my $target_mtt = $stream->pos($target->p_start);
+            $target_mtt->add(
+              term => '<:mate/d:' . $label,
+              pti => 34, # element-to-term relation
+              payload =>
+                '<i>' . $target->o_start . # end position
+                '<i>' . $target->o_end . # end position
+                '<i>' . $target->p_end . # right part end position
+                '<i>' . $source->pos # . # left part token position
+                #      '<s>0' . # $source_term->tui . # left part tui
+                #        '<s>0' # . $target_span->tui # right part tui
 
-	    );
-	  }
-	  else {
-	    #	    use Data::Dumper;
-	    #	    warn '2###### ' . Dumper($content);
-	  };
-	};
+              );
+          }
+          else {
+            #      use Data::Dumper;
+            #      warn '2###### ' . Dumper($content);
+          };
+        };
       };
     }) or return;
 
diff --git a/lib/KorAP/XML/Tokenizer/Range.pm b/lib/KorAP/XML/Tokenizer/Range.pm
index 366585c..8b3e992 100644
--- a/lib/KorAP/XML/Tokenizer/Range.pm
+++ b/lib/KorAP/XML/Tokenizer/Range.pm
@@ -80,18 +80,21 @@
     return $2;
   }
   else {
-    # I am not sure about that ...
-    $found;
+    # The current position is likely wrong. The most common non-gap
+    # and non-exact-match is in the situation of
+    # "y<e> z</e>", where e is in the range of the y token.
+    $found + 1;
   };
 };
 
 
 sub to_string {
   my $self = shift;
-  return join('',
-	      map {'['.join(',',@$_).']'}
-		@{$$self->get_range(0,100,'...')}
-	      ) . '...';
+  return join(
+    '',
+    map {'['.join(',',@$_).']'}
+      @{$$self->get_range(0,100,'...')}
+    ) . '...';
 };
 
 1;
diff --git a/lib/KorAP/XML/Tokenizer/Span.pm b/lib/KorAP/XML/Tokenizer/Span.pm
index b359e14..7d7bd55 100644
--- a/lib/KorAP/XML/Tokenizer/Span.pm
+++ b/lib/KorAP/XML/Tokenizer/Span.pm
@@ -2,6 +2,7 @@
 use strict;
 use warnings;
 use Mojo::DOM;
+use Clone;
 
 sub new {
   bless [], shift;
@@ -109,4 +110,11 @@
   };
 };
 
+
+# Clone the span
+sub clone {
+  return Clone::clone(shift);
+};
+
+
 1;
diff --git a/lib/KorAP/XML/Tokenizer/Units.pm b/lib/KorAP/XML/Tokenizer/Units.pm
index 6b75e42..22f50d5 100644
--- a/lib/KorAP/XML/Tokenizer/Units.pm
+++ b/lib/KorAP/XML/Tokenizer/Units.pm
@@ -8,6 +8,8 @@
 has 'have' => 0;
 has 'encoding' => 'utf-8';
 
+use constant DEBUG => 0;
+
 sub span {
   my $self = shift;
   my ($from, $to, $s) = @_;
@@ -19,11 +21,15 @@
   $from //= 0;
 
   # The span is invalid
-  return unless $from <= $to;
+  unless ($from <= $to) {
+    if (DEBUG) {
+      warn $s->{-id} . ' is invalid';
+    };
+    return;
+  };
 
   my $span = KorAP::XML::Tokenizer::Span->new;
 
-
   # The span is a milestone
   if ($from == $to) {
     $span->milestone(1);
@@ -41,7 +47,13 @@
 
   unless (defined $start) {
     $start = $self->range->after($span->o_start);
-    return unless defined $start;
+
+    unless (defined $start) {
+      if (DEBUG) {
+        warn $span->id . ' has no valid start';
+      };
+      return;
+    };
   };
 
   # Set start token position to span
@@ -57,25 +69,45 @@
 
     unless (defined $end) {
       $end = $self->range->before($span->o_end);
-      return unless defined $end;
+
+      unless (defined $end) {
+        if (DEBUG) {
+          warn $span->id . ' has no valid end';
+        };
+        return;
+      };
 
       # The next token of end has a character
-      # offset AFTER th given end character offset
+      # offset AFTER the given end character offset
       my $real_start = $self->stream->pos($end)->o_start;
 
       # Ignore non-milestone elements outside the token stream!
-      return if $to <= $real_start;
+      if ($to <= $real_start) {
+        if (DEBUG) {
+          warn 'Ignore ' . $span->id . ' is a non-milestone element outside the token stream';
+        };
+        return;
+      };
     };
 
     # $span->p_end($end);
     # return unless $span->p_end >= $span->p_start;
 
     # EXPERIMENTAL:
-    return unless $end >= $span->p_start;
+    unless ($end >= $span->p_start) {
+      if (DEBUG) {
+        warn 'Ignore ' . $span->id . ' with ' . $span->p_start . '-' . $end;
+      };
+      return;
+    };
 
     $span->p_end($end + 1);
   }
 
+  if (DEBUG && $from == 124) {
+    warn 'exact: ' . $span->p_start . '-' . $span->p_end;
+  };
+
   $span->hash($s) if $s;
 
   $span;
@@ -116,6 +148,8 @@
     $from = $p->bytes2chars($from);
     $to = $p->bytes2chars($to);
   }
+
+  # This is legacy treating of bytes2chars
   elsif ($self->encoding eq 'xip') {
     $from = $p->xip2chars($from);
     $to = $p->xip2chars($to);
diff --git a/t/annotation/dereko_struct.t b/t/annotation/dereko_struct.t
index 5569369..4e00bb3 100644
--- a/t/annotation/dereko_struct.t
+++ b/t/annotation/dereko_struct.t
@@ -42,19 +42,19 @@
    '@:dereko/s:pattern:text$<b>17<s>2',
    'Attribute of idsHeader');
 
-is($data->{stream}->[4]->[1],
+is($data->{stream}->[5]->[1],
    '<>:dereko/s:s$<b>64<i>32<i>42<i>6<b>6<s>1',
    'Sentence span');
 
-is($data->{stream}->[4]->[2],
+is($data->{stream}->[5]->[2],
    '@:dereko/s:broken:no$<b>17<s>1<i>6',
    'Attribute of sentence span');
 
-is($data->{stream}->[6]->[0],
-   '<>:dereko/s:pb$<b>65<i>42<i>42<i>6<b>6<s>1',
+is($data->{stream}->[7]->[0],
+   '<>:dereko/s:pb$<b>65<i>42<i>42<i>7<b>6<s>1',
    'Pagebreak element');
 
-is($data->{stream}->[6]->[-1],
+is($data->{stream}->[7]->[-1],
    '~:base/s:pb$<i>2<i>42',
    'Pagebreak element');
 
diff --git a/t/corpus/REDEW/DOC1/00000/data.xml b/t/corpus/REDEW/DOC1/00000/data.xml
new file mode 100644
index 0000000..3b85204
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/data.xml
@@ -0,0 +1,7 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="text.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<raw_text docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP">
+  <metadata file="metadata.xml" />
+  <text>Cechov,_Anton_Pavlovic_Gram.tg4_1.xml 1886 1880 Gram Čechov, Anton Pavlovič yes yes Erzähltext digbib no third chevron ... Hörst du, du alter Drachen? Oder machst du dir nichts aus unseren Worten?«</text>
+</raw_text>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml b/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml
new file mode 100644
index 0000000..66f7ba1
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/drukola/morpho.xml
@@ -0,0 +1,205 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+  <spanList>
+    <span id="s0" from="119" to="122" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">--</f>
+            <f name="pos">$(</f>
+            <f name="msd">msd=rfpos=SYM.Pun.Cont|sentstart=yes|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s1" from="123" to="128" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">Hörst</f>
+            <f name="pos">VVFIN</f>
+            <f name="msd">msd=rfpos=VFIN.Full.2.Sg.Pres.Ind|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s2" from="129" to="131" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">du</f>
+            <f name="pos">PPER</f>
+            <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s3" from="131" to="132" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">--</f>
+            <f name="pos">$,</f>
+            <f name="msd">msd=rfpos=SYM.Pun.Comma|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s4" from="133" to="135" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">du</f>
+            <f name="pos">PPER</f>
+            <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s5" from="136" to="141" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">alt</f>
+            <f name="pos">ADJA</f>
+            <f name="msd">msd=rfpos=ADJA.Pos.Gen.Pl.Masc|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s6" from="142" to="149" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">Drache</f>
+            <f name="pos">NN</f>
+            <f name="msd">msd=rfpos=N.Reg.Gen.Pl.Masc|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s7" from="149" to="150" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">--</f>
+            <f name="pos">$.</f>
+            <f name="msd">msd=rfpos=SYM.Pun.Sent|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s8" from="151" to="155" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">oder</f>
+            <f name="pos">NE</f>
+            <f name="msd">msd=rfpos=CONJ.Coord.-|sentstart=yes|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s9" from="156" to="162" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">machen</f>
+            <f name="pos">VVFIN</f>
+            <f name="msd">msd=rfpos=VFIN.Full.2.Sg.Pres.Ind|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s10" from="163" to="165" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">du</f>
+            <f name="pos">PPER</f>
+            <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Nom.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s11" from="166" to="169" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">dir</f>
+            <f name="pos">PPER</f>
+            <f name="msd">msd=rfpos=PRO.Pers.Subst.2.Dat.Sg.*|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s12" from="170" to="176" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">nichts</f>
+            <f name="pos">PIS</f>
+            <f name="msd">msd=rfpos=PRO.Indef.Subst.-.*.*.Neut|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s13" from="177" to="180" l="6">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">aus</f>
+            <f name="pos">APPR</f>
+            <f name="msd">msd=rfpos=APPR.Dat|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s14" from="181" to="188" l="7">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">unser</f>
+            <f name="pos">PPOSAT</f>
+            <f name="msd">msd=rfpos=PRO.Poss.Attr.-.Dat.Pl.Neut|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s15" from="189" to="195" l="7">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">Wort</f>
+            <f name="pos">NN</f>
+            <f name="msd">msd=rfpos=N.Reg.Dat.Pl.Neut|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s16" from="195" to="196" l="6">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">--</f>
+            <f name="pos">$.</f>
+            <f name="msd">msd=rfpos=SYM.Pun.Sent|sentstart=no|stwr=direct.speech.1|reported.speech.19</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s17" from="196" to="197" l="5">
+      <fs type="lex" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="lex">
+          <fs>
+            <f name="lemma">"</f>
+            <f name="pos">$(</f>
+            <f name="msd">msd=rfpos=N.Reg.Nom.Sg.Masc|sentstart=no|stwr=direct.speech.1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+  </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/header.xml b/t/corpus/REDEW/DOC1/00000/header.xml
new file mode 100644
index 0000000..9738cd3
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/header.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="text" pattern="text" status="new" version="1.1" TEIform="teiHeader">
+       <fileDesc>
+         <titleStmt>
+            <title>Gram</title>
+<!--- missing in origin data -->
+<textSigle>REDEW/DOC1.00000</textSigle>
+<!-- -->
+             <author>Čechov, Anton Pavlovič</author>
+            <respStmt>
+               <resp></resp>
+               <name></name>
+            </respStmt>
+         </titleStmt>
+         <publicationStmt>
+            <distributor>www.redewiedergabe.de</distributor>
+         </publicationStmt>
+         <sourceDesc>
+            <bibl></bibl>
+         </sourceDesc>
+       </fileDesc>
+   </idsHeader>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/00000/struct/structure.xml b/t/corpus/REDEW/DOC1/00000/struct/structure.xml
new file mode 100644
index 0000000..c12f102
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/00000/struct/structure.xml
@@ -0,0 +1,200 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="span.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+
+<layer docid="REDEW_DOC1.00000" xmlns="http://ids-mannheim.de/ns/KorAP" version="KorAP-0.4">
+  <spanList>
+    <span id="s0" from="0" to="197" l="1">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">text</f>
+      </fs>
+    </span>
+    <span id="s1" from="0" to="118" l="2">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">fs</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="type">metadata</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s2" from="0" to="37" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">filename</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s3" from="38" to="42" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">year</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s4" from="43" to="47" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">decade</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s5" from="48" to="52" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">title</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s6" from="53" to="75" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">author</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s7" from="76" to="79" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">fictional</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s8" from="80" to="83" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">narrative</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s9" from="84" to="94" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">textype</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s10" from="95" to="101" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">source</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s11" from="102" to="104" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">dialect</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s12" from="105" to="110" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">perspective</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s13" from="111" to="118" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">f</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="name">quotes</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s14" from="118" to="197" l="2">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">body</f>
+      </fs>
+    </span>
+    <span id="s15" from="118" to="197" l="3">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">p</f>
+      </fs>
+    </span>
+    <span id="s16" from="118" to="197" l="4">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">said</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="id">1</f>
+            <f name="mode">direct</f>
+            <f name="content">speech</f>
+            <f name="level">1</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s30" from="176" to="196" l="5">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">said</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="id">19</f>
+            <f name="mode">reported</f>
+            <f name="content">speech</f>
+            <f name="level">2</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s32" from="180" to="188" l="6">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">seg</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="id">19</f>
+            <f name="type">speaker</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+    <span id="s34" from="188" to="195" l="6">
+      <fs type="struct" xmlns="http://www.tei-c.org/ns/1.0">
+        <f name="name">seg</f>
+        <f name="attr">
+          <fs type="attr">
+            <f name="id">19</f>
+            <f name="type">intExpr</f>
+          </fs>
+        </f>
+      </fs>
+    </span>
+  </spanList>
+</layer>
\ No newline at end of file
diff --git a/t/corpus/REDEW/DOC1/header.xml b/t/corpus/REDEW/DOC1/header.xml
new file mode 100644
index 0000000..b64c4db
--- /dev/null
+++ b/t/corpus/REDEW/DOC1/header.xml
@@ -0,0 +1,44 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="document" pattern="text" status="new" version="1.1" TEIform="teiHeader">
+   <fileDesc>
+    <titleStmt>
+<!--- missing in origin data -->
+     <dokumentSigle>REDEW/DOC1</dokumentSigle>
+<!-- -->
+     <d.title>Redewiedergabe Dokument 1</d.title>
+    </titleStmt>
+    <publicationStmt>
+     <distributor/>
+     <pubAddress/>
+     <availability region="world">QAO-NC</availability>
+     <pubDate/>
+    </publicationStmt>
+    <sourceDesc>
+     <biblStruct>
+      <monogr>
+       <h.title type="main"/>
+       <edition>
+        <further/>
+        <kind/>
+        <appearance/>
+       </edition>
+       <imprint/>
+       <biblNote n="1">?</biblNote>
+      </monogr>
+     </biblStruct>
+    </sourceDesc>
+   </fileDesc>
+   <profileDesc>
+    <creation>
+     <creatDate>2020</creatDate>
+     <creatRef>?</creatRef>
+     <creatRefShort>?</creatRefShort>
+    </creation>
+    <textDesc>
+     <textType>?</textType>
+     <textTypeRef/>
+    </textDesc>
+   </profileDesc>
+  </idsHeader>
\ No newline at end of file
diff --git a/t/corpus/REDEW/header.xml b/t/corpus/REDEW/header.xml
new file mode 100644
index 0000000..dc35127
--- /dev/null
+++ b/t/corpus/REDEW/header.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-model href="header.rng" type="application/xml" schematypens="http://relaxng.org/ns/structure/1.0"?>
+<!DOCTYPE idsCorpus PUBLIC "-//IDS//DTD IDS-XCES 1.0//EN" "http://corpora.ids-mannheim.de/idsxces1/DTD/ids.xcesdoc.dtd">
+<idsHeader type="corpus" pattern="allesaußerZtg/Zschr" status="new" version="1.1" TEIform="teiHeader">
+  <fileDesc>
+   <titleStmt>
+<!--- missing in origin data -->
+    <korpusSigle>REDEW</korpusSigle>
+<!-- -->
+    <c.title>Redewiedergabe</c.title>
+   </titleStmt>
+   <publicationStmt>
+    <distributor>		Institut für Deutsche Sprache		</distributor>
+    <pubAddress>		Postfach 10 16 21, D-68016 Mannheim	</pubAddress>
+    <telephone>		+49 (0)621 1581 0			</telephone>
+    <availability region="world">QAO-NC</availability>
+    <pubDate/>
+   </publicationStmt>
+   <sourceDesc>
+    <biblFull>
+     <titleStmt>
+      <x.title/>
+     </titleStmt>
+     <editionStmt/>
+     <publicationStmt>
+      <distributor/>
+      <pubAddress/>
+      <availability region="world">QAO-NC</availability>
+      <pubDate/>
+     </publicationStmt>
+    </biblFull>
+    <biblStruct>
+     <monogr>
+      <h.title type="main">Redewiedergabe</h.title>
+      <h.title type="sub"/>
+      <h.author>...</h.author>
+      <editor>...</editor>
+      <edition>
+       <further/>
+       <kind></kind>
+       <appearance/>
+      </edition>
+      <imprint>
+       <publisher>...</publisher>
+       <pubDate type="year">2020</pubDate>
+       <pubDate type="month"/>
+       <pubDate type="day"/>
+       <pubPlace key="DE"></pubPlace>
+      </imprint>
+     </monogr>
+    </biblStruct>
+    <reference type="super" assemblage="non-automatic">...</reference>
+   </sourceDesc>
+  </fileDesc>
+ </idsHeader>
\ No newline at end of file
diff --git a/t/range.t b/t/range.t
index 27c7970..2091661 100644
--- a/t/range.t
+++ b/t/range.t
@@ -44,8 +44,8 @@
 is($range->after(0), 0, 'After is fine');
 is($range->after(1), 0, 'After is fine');
 is($range->after(2), 0, 'After is fine');
-is($range->after(3), 0, 'After is fine');
-is($range->after(14), 0, 'After is fine');
+is($range->after(3), 1, 'After is fine');
+is($range->after(14), 1, 'After is fine');
 is($range->after(15), 1, 'After is fine');
 
 is($range->to_string,
diff --git a/t/real/agd.t b/t/real/agd.t
index 6ba7ebe..7745bea 100644
--- a/t/real/agd.t
+++ b/t/real/agd.t
@@ -118,12 +118,12 @@
 $first_token = join('||', @{$output->{data}->{stream}->[0]});
 like($first_token, qr!<>:base/s:s\$<b>64<i>0<i>16<i>2<b>1!);
 
-my $token = join('||', @{$output->{data}->{stream}->[1]});
+my $token = join('||', @{$output->{data}->{stream}->[2]});
 like($token, qr!<>:base/s:s\$<b>64<i>16<i>23<i>4<b>1!);
-$token = join('||', @{$output->{data}->{stream}->[2]});
+$token = join('||', @{$output->{data}->{stream}->[3]});
 unlike($token, qr!<>:base/s:s!);
 
-$token = join('||', @{$output->{data}->{stream}->[3]});
+$token = join('||', @{$output->{data}->{stream}->[4]});
 like($token, qr!<>:base/s:s\$<b>64<i>23<i>27<i>5<b>1!);
 
 $token = join('||', @{$output->{data}->{stream}->[5]});
diff --git a/t/real/redew.t b/t/real/redew.t
new file mode 100644
index 0000000..25f369c
--- /dev/null
+++ b/t/real/redew.t
@@ -0,0 +1,155 @@
+use strict;
+use warnings;
+use Test::More;
+use Data::Dumper;
+use JSON::XS;
+use Log::Log4perl;
+
+use utf8;
+use lib 'lib', '../lib';
+
+use File::Basename 'dirname';
+use File::Spec::Functions 'catdir';
+
+# Initialize log4perl object
+Log::Log4perl->init({
+  'log4perl.rootLogger' => 'ERROR, STDERR',
+  'log4perl.appender.STDERR' => 'Log::Log4perl::Appender::ScreenColoredLevels',
+  'log4perl.appender.STDERR.layout' => 'PatternLayout',
+  'log4perl.appender.STDERR.layout.ConversionPattern' => '[%r] %F %L %c - %m%n'
+});
+
+use_ok('KorAP::XML::Krill');
+
+my $path = catdir(dirname(__FILE__), '../corpus/REDEW/DOC1/00000');
+
+ok(my $doc = KorAP::XML::Krill->new( path => $path . '/' ), 'Load Korap::Document');
+ok($doc->parse, 'Parse document');
+
+is($doc->text_sigle, 'REDEW/DOC1/00000', 'Correct text sigle');
+is($doc->doc_sigle, 'REDEW/DOC1', 'Correct document sigle');
+is($doc->corpus_sigle, 'REDEW', 'Correct corpus sigle');
+
+my $meta = $doc->meta;
+ok(!$meta->{T_title}, 'Title');               # ???
+ok(!$meta->{T_sub_title}, 'SubTitle');
+ok(!$meta->{T_author}, 'Author');
+ok(!$meta->{A_editor}, 'Editor');
+ok(!$meta->{S_pub_place}, 'PubPlace');
+is($meta->{A_publisher}, '...', 'Publisher'); # ???
+
+is($meta->{S_text_type}, '?', 'Text Type');   # ???
+ok(!$meta->{S_text_type_art}, 'No Text Type Art');
+ok(!$meta->{S_text_type_ref}, 'No Text Type Ref');
+ok(!$meta->{S_text_domain}, 'No Text Domain');
+ok(!$meta->{S_text_column}, 'No Text Column');
+
+ok(!$meta->{K_text_class}->[0], 'Correct Text Class');
+
+is($meta->{D_pub_date}, '20200000', 'Creation date');
+is($meta->{D_creation_date}, '20200000', 'Creation date');
+is($meta->{S_availability}, 'QAO-NC', 'License');           # ???
+ok(!$meta->{A_pages}, 'Pages');
+
+ok(!$meta->{A_file_edition_statement}, 'File Statement');
+ok(!$meta->{A_bibl_edition_statement}, 'Bibl Statement');
+
+ok(!$meta->{A_reference}, 'Reference');
+ok(!$meta->{S_language}, 'Language'); # ???
+
+is($meta->{T_corpus_title}, 'Redewiedergabe', 'Correct Corpus title');
+ok(!$meta->{T_corpus_sub_title}, 'Correct Corpus sub title');
+is($meta->{T_corpus_author}, '...', 'Correct Corpus author'); # ???
+is($meta->{A_corpus_editor}, '...', 'Correct Corpus editor'); # ???
+
+is($meta->{T_doc_title}, 'Redewiedergabe Dokument 1', 'Correct Doc title');
+ok(!$meta->{T_doc_sub_title}, 'Correct Doc sub title');
+ok(!$meta->{T_doc_author}, 'Correct Doc author');
+ok(!$meta->{A_doc_editor}, 'Correct doc editor');
+
+# Tokenization
+use_ok('KorAP::XML::Tokenizer');
+
+my ($token_base_foundry, $token_base_layer) = (qw/drukola Morpho/);
+
+# Get tokenization
+my $tokens = KorAP::XML::Tokenizer->new(
+  path => $doc->path,
+  doc => $doc,
+  foundry => $token_base_foundry,
+  layer => $token_base_layer,
+  name => 'tokens'
+);
+
+ok($tokens, 'Token Object is fine');
+ok($tokens->parse, 'Token parsing is fine');
+
+my $output = decode_json( $tokens->to_json );
+
+is(substr($output->{data}->{text}, 0, 100), 'Cechov,_Anton_Pavlovic_Gram.tg4_1.xml 1886 1880 Gram Čechov, Anton Pavlovič yes yes Erzähltext digbi', 'Primary Data');
+
+is($output->{data}->{name}, 'tokens', 'tokenName');
+is($output->{data}->{tokenSource}, 'drukola#morpho', 'tokenSource');
+is($output->{version}, '0.03', 'version');
+
+is($output->{data}->{foundries}, '', 'Foundries');
+is($output->{data}->{layerInfos}, '', 'layerInfos');
+is($output->{data}->{stream}->[0]->[4], 's:Hörst', 'data');
+
+is($output->{textSigle}, 'REDEW/DOC1/00000', 'Correct text sigle');
+is($output->{docSigle}, 'REDEW/DOC1', 'Correct document sigle');
+is($output->{corpusSigle}, 'REDEW', 'Correct corpus sigle');
+
+ok(!$output->{title}, 'Title');
+ok(!$output->{subTitle}, 'Correct SubTitle');
+ok(!$output->{author}, 'Author');
+ok(!exists $output->{editor}, 'Publisher');
+
+# Add annotations
+$tokens->add('DRuKoLa', 'Morpho');
+$tokens->add('DeReKo', 'Structure');
+
+$output = decode_json( $tokens->to_json );
+
+my $first = $output->{data}->{stream}->[0];
+
+is('-:tokens$<i>13',$first->[0]);
+is('<>:base/s:t$<b>64<i>0<i>197<i>12<b>0',$first->[1]);
+is('<>:dereko/s:text$<b>64<i>0<i>197<i>12<b>0',$first->[2]);
+is('<>:dereko/s:body$<b>64<i>118<i>197<i>12<b>1',$first->[3]);
+is('<>:dereko/s:p$<b>64<i>118<i>197<i>12<b>2',$first->[4]);
+is('<>:dereko/s:said$<b>64<i>118<i>197<i>12<b>3<s>1',$first->[5]);
+is('@:dereko/s:level:1$<b>17<s>1<i>12',$first->[6]);
+is('@:dereko/s:content:speech$<b>17<s>1<i>12',$first->[7]);
+is('@:dereko/s:mode:direct$<b>17<s>1<i>12',$first->[8]);
+is('@:dereko/s:id:1$<b>17<s>1<i>12',$first->[9]);
+is('_0$<i>123<i>128',$first->[10]);
+is("drukola/l:H\x{f6}rst",$first->[11]);
+is('drukola/m:msd:rfpos',$first->[12]);
+is('drukola/m:sentstart:no',$first->[13]);
+is('drukola/m:stwr:direct.speech.1',$first->[14]);
+is('drukola/p:VVFIN',$first->[15]);
+is("i:h\x{f6}rst",$first->[16]);
+is("s:H\x{f6}rst",$first->[17]);
+
+my $nine = join(',', @{$output->{data}->{stream}->[9]});
+like($nine, qr{drukola\/l:nichts}, 'Nichts');
+like($nine, qr{_9\$<i>170<i>176}, 'Term boundaries');
+unlike($nine, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>12<b>4<s>1}, 'Term boundaries');
+
+my $ten = join(',', @{$output->{data}->{stream}->[10]});
+like($ten, qr{_10\$<i>177<i>180}, 'Term boundaries');
+like($ten, qr{<>:dereko/s:said\$<b>64<i>176<i>196<i>12<b>4<s>1}, 'Term boundaries');
+
+my $eleven = join(',', @{$output->{data}->{stream}->[11]});
+like($eleven, qr{_11\$<i>181<i>188}, 'Term boundaries');
+like($eleven, qr{<>:dereko/s:seg\$<b>64<i>180<i>188<i>12<b>5<s>1}, 'Segment');
+
+
+my $twelve = join(',', @{$output->{data}->{stream}->[12]});
+like($twelve, qr{_12\$<i>189<i>195}, 'Term boundaries');
+like($twelve, qr{drukola/l:Wort}, 'Lemma');
+like($twelve, qr{<>:dereko/s:seg\$<b>64<i>188<i>195<i>13<b>5<s>1}, 'Segment');
+
+done_testing;
+__END__
diff --git a/t/real/rei.t b/t/real/rei.t
index b375c24..d50a732 100644
--- a/t/real/rei.t
+++ b/t/real/rei.t
@@ -225,16 +225,17 @@
 
 my $last = $output->{data}->{stream}->[-1];
 
-is('<>:dereko/s:text$<b>65<i>17859<i>17859<i>2640<b>0', $last->[0]);
-is('<>:dereko/s:back$<b>65<i>17859<i>17859<i>2640<b>1', $last->[1]);
-is('>:malt/d:APP$<b>32<i>2639', $last->[2]);
-is('_2640$<i>17851<i>17859', $last->[3]);
-is('corenlp/p:NE', $last->[4]);
-is("i:schr\x{f6}der", $last->[5]);
-is('opennlp/p:NE', $last->[6]);
-is("s:Schr\x{f6}der", $last->[7]);
-is("tt/l:Schr\x{f6}der", $last->[8]);
-is('tt/p:NE', $last->[9]);
+# Milestones behind the final token are no longer indexed
+# is('<>:dereko/s:text$<b>65<i>17859<i>17859<i>2640<b>0', $last->[0]);
+# is('<>:dereko/s:back$<b>65<i>17859<i>17859<i>2640<b>1', $last->[1]);
+is('>:malt/d:APP$<b>32<i>2639', $last->[0]);
+is('_2640$<i>17851<i>17859', $last->[1]);
+is('corenlp/p:NE', $last->[2]);
+is("i:schr\x{f6}der", $last->[3]);
+is('opennlp/p:NE', $last->[4]);
+is("s:Schr\x{f6}der", $last->[5]);
+is("tt/l:Schr\x{f6}der", $last->[6]);
+is('tt/p:NE', $last->[7]);
 
 
 # REI/BNG/00071
@@ -282,7 +283,7 @@
 is('-:corenlp/sentences$<i>69', $first->[2]);
 is('-:opennlp/sentences$<i>65', $first->[3]);
 is('-:tokens$<i>1009', $first->[4]);
-is('-:tt/sentences$<i>95', $first->[5]);
+is('-:tt/sentences$<i>94', $first->[5]);
 is('<:malt/d:APP$<b>32<i>1', $first->[6]);
 is('<>:corenlp/c:MPN$<b>64<i>0<i>16<i>2<b>3', $first->[7]);
 is('<>:base/s:s$<b>64<i>0<i>48<i>5<b>2', $first->[8]);
diff --git a/t/real/wpd.t b/t/real/wpd.t
index 517f9be..e45db0a 100644
--- a/t/real/wpd.t
+++ b/t/real/wpd.t
@@ -78,10 +78,17 @@
 my $stream = $tokens->to_data->{data}->{stream};
 
 # This is not a goot relation example
-is($stream->[77]->[0],
-   '<:mate/d:--$<b>34<i>498<i>499<i>78<i>78',
-   'element to term');
-is($stream->[78]->[0], '>:mate/d:--$<b>33<i>498<i>499<i>77<i>78', 'term to element');
+ is($stream->[79]->[0],
+    '>:mate/d:CJ$<b>32<i>68',
+    'term to term');
+ is($stream->[79]->[1], '<:mate/d:PD$<b>32<i>81', 'term to term');
+
+
+# These are no longer aligned
+# is($stream->[77]->[0],
+#    '<:mate/d:--$<b>34<i>498<i>499<i>78<i>78',
+#    'element to term');
+# is($stream->[78]->[0], '>:mate/d:--$<b>33<i>498<i>499<i>77<i>78', 'term to element');
 
 $tokens->add('Base', 'Sentences');
 
diff --git a/t/real/wpf.t b/t/real/wpf.t
index 4a28c2f..ba6fb94 100644
--- a/t/real/wpf.t
+++ b/t/real/wpf.t
@@ -101,10 +101,11 @@
 $tokens->add('Talismane', 'Morpho');
 
 $stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[10], 'talismane/l:Sigmund', 'Talismane morpho annotation');
-is($stream->[1]->[11], 'talismane/m:g:m', 'Talismane morpho annotation');
-is($stream->[1]->[12], 'talismane/m:n:s', 'Talismane morpho annotation');
-is($stream->[1]->[13], 'talismane/p:NPP', 'Talismane morpho annotation');
+
+is($stream->[1]->[9], 'talismane/l:Sigmund', 'Talismane morpho annotation');
+is($stream->[1]->[10], 'talismane/m:g:m', 'Talismane morpho annotation');
+is($stream->[1]->[11], 'talismane/m:n:s', 'Talismane morpho annotation');
+is($stream->[1]->[12], 'talismane/p:NPP', 'Talismane morpho annotation');
 
 is($stream->[300]->[5], 'talismane/l:son', 'Talismane lemma annotation');
 is($stream->[300]->[6], 'talismane/m:g:m', 'Talismane morph annotation');
@@ -116,7 +117,10 @@
 # Add Malt dependency
 $tokens->add('Malt', 'Dependency');
 $stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[1], '>:malt/d:dep$<b>33<i>7<i>8<i>0<i>1', 'Malt dep annotation');
+
+# This is no longer indexed
+# is($stream->[1]->[1], '>:malt/d:dep$<b>33<i>7<i>8<i>0<i>1', 'Malt dep annotation');
+
 is($stream->[1]->[2], '<:malt/d:dep$<b>32<i>2', 'Malt dep annotation');
 is($stream->[300]->[1], '>:malt/d:punct$<b>32<i>302', 'Malt dep annotation');
 
@@ -124,9 +128,10 @@
 # Add TreeTagger morpho
 $tokens->add('TreeTagger', 'Morpho');
 $stream = $tokens->to_data->{data}->{stream};
-is($stream->[1]->[14], 'tt/p:ADJ$<b>129<b>26', 'TreeTagger morph annotation');
-is($stream->[1]->[15], 'tt/p:NAM$<b>129<b>200', 'TreeTagger morph annotation');
-is($stream->[1]->[16], 'tt/p:NOM$<b>129<b>27', 'TreeTagger morph annotation');
+
+is($stream->[1]->[13], 'tt/p:ADJ$<b>129<b>26', 'TreeTagger morph annotation');
+is($stream->[1]->[14], 'tt/p:NAM$<b>129<b>200', 'TreeTagger morph annotation');
+is($stream->[1]->[15], 'tt/p:NOM$<b>129<b>27', 'TreeTagger morph annotation');
 
 
 done_testing;
diff --git a/t/script/non_verbal_tokens.t b/t/script/non_verbal_tokens.t
index fa460e4..5327523 100644
--- a/t/script/non_verbal_tokens.t
+++ b/t/script/non_verbal_tokens.t
@@ -54,7 +54,7 @@
 my $token = $stream->[4];
 is($token->[3], 'dgd/l:pui', 'Token');
 $token = $stream->[5];
-is($token->[13], 'dgd/l:xui', 'Token');
+is($token->[15], 'dgd/l:xui', 'Token');
 
 $call = join(
   ' ',
@@ -88,10 +88,10 @@
 is($token->[3], 'dgd/l:pui', 'Token');
 
 $token = $stream->[5];
-is($token->[14], 'dgd/para:pause$<b>128<s>5', 'Token');
+is($token->[4], 'dgd/para:pause$<b>128<s>2', 'Token');
 
 $token = $stream->[6];
-is($token->[1], 'dgd/l:xui', 'Token');
+is($token->[13], 'dgd/l:xui', 'Token');
 
 
 
diff --git a/t/script/non_word_tokens.t b/t/script/non_word_tokens.t
index 0eb9e14..b156866 100644
--- a/t/script/non_word_tokens.t
+++ b/t/script/non_word_tokens.t
@@ -51,7 +51,7 @@
 is($json->{data}->{foundries}, 'base base/paragraphs base/sentences connexor connexor/morpho connexor/phrase connexor/sentences connexor/syntax corenlp corenlp/sentences dereko dereko/structure mate mate/dependency mate/morpho opennlp opennlp/morpho opennlp/sentences treetagger treetagger/morpho treetagger/sentences xip xip/constituency xip/morpho xip/sentences', 'Foundries');
 my $stream = $json->{data}->{stream};
 my $token = $stream->[12];
-is($token->[17], 's:Vokal', 'Token');
+is($token->[16], 's:Vokal', 'Token');
 $token = $stream->[13];
 is($token->[23], 's:Der', 'Token');
 
diff --git a/t/transform.t b/t/transform.t
index 4537bcb..97c5a8b 100644
--- a/t/transform.t
+++ b/t/transform.t
@@ -154,7 +154,7 @@
   '<>:opennlp/s:s$<b>64<i>0<i>74<i>13<b>0|'.
   '-:opennlp/sentences$<i>50|'.
   '<>:corenlp/s:s$<b>64<i>0<i>6<i>2<b>0|'.
-  '-:corenlp/sentences$<i>68|'.
+  '-:corenlp/sentences$<i>67|'.
   'cnx/l:A|'.
   'cnx/p:N|'.
   'cnx/syn:@NH|'.