Optimize annotations to use term based Multiterm construction

Change-Id: I57d1cd0b8314db6447538ddf9e01eb6aa0d4cb9f
diff --git a/Changes b/Changes
index e2092a3..1f88c45 100644
--- a/Changes
+++ b/Changes
@@ -1,10 +1,11 @@
-0.41 2020-08-03
+0.41 2020-08-04
         - Added support for RWK annotations.
         - Improved DGD support.
         - Fixed bug in RWK support that broke on
           some KorAP-XML files.
         - Separate "real data" test suite from artificial
           tests to prepare for CPAN release.
+        - Optimizations and cleanup based on profiling.
 
 0.40 2020-03-03
         - Fixed XIP parser.
diff --git a/lib/KorAP/XML/Annotation/Base/Paragraphs.pm b/lib/KorAP/XML/Annotation/Base/Paragraphs.pm
index 40a37d2..922df0c 100644
--- a/lib/KorAP/XML/Annotation/Base/Paragraphs.pm
+++ b/lib/KorAP/XML/Annotation/Base/Paragraphs.pm
@@ -11,7 +11,7 @@
     cb => sub {
       my ($stream, $span) = @_;
       my $mtt = $stream->pos($span->get_p_start);
-      my $mt = $mtt->add('<>:base/s:p');
+      my $mt = $mtt->add_by_term('<>:base/s:p');
       $mt->set_o_start($span->get_o_start);
       $mt->set_o_end($span->get_o_end);
       $mt->set_p_end($span->get_p_end);
diff --git a/lib/KorAP/XML/Annotation/Base/Sentences.pm b/lib/KorAP/XML/Annotation/Base/Sentences.pm
index 3c9b3ce..9ccdca6 100644
--- a/lib/KorAP/XML/Annotation/Base/Sentences.pm
+++ b/lib/KorAP/XML/Annotation/Base/Sentences.pm
@@ -16,7 +16,7 @@
 
       $first = [$span->get_p_start, $span->get_o_start] unless defined $first;
 
-      my $mt = $mtt->add('<>:base/s:s');
+      my $mt = $mtt->add_by_term('<>:base/s:s');
       $mt->set_o_start($span->get_o_start);
       $mt->set_o_end($span->get_o_end);
       $mt->set_p_end($span->get_p_end);
diff --git a/lib/KorAP/XML/Annotation/CoreNLP/Constituency.pm b/lib/KorAP/XML/Annotation/CoreNLP/Constituency.pm
index 0138594..3f33802 100644
--- a/lib/KorAP/XML/Annotation/CoreNLP/Constituency.pm
+++ b/lib/KorAP/XML/Annotation/CoreNLP/Constituency.pm
@@ -59,7 +59,7 @@
     my $type = $f->{'#text'} or return;
 
     # $type is now NPA, NP, NUM ...
-    my $term = $mtt->add('<>:corenlp/c:' . $type);
+    my $term = $mtt->add_by_term('<>:corenlp/c:' . $type);
     $term->set_o_start($span->get_o_start);
     $term->set_o_end($span->get_o_end);
     $term->set_p_end($span->get_p_end);
diff --git a/lib/KorAP/XML/Annotation/CoreNLP/Morpho.pm b/lib/KorAP/XML/Annotation/CoreNLP/Morpho.pm
index 97a139f..228789c 100644
--- a/lib/KorAP/XML/Annotation/CoreNLP/Morpho.pm
+++ b/lib/KorAP/XML/Annotation/CoreNLP/Morpho.pm
@@ -16,9 +16,7 @@
 
       # syntax
       if (($content->{-name} eq 'pos') && ($content->{'#text'})) {
-        $mtt->add(
-          term => 'corenlp/p:' . $content->{'#text'}
-        );
+        $mtt->add_by_term('corenlp/p:' . $content->{'#text'});
       };
     }) or return;
 
diff --git a/lib/KorAP/XML/Annotation/CoreNLP/NamedEntities.pm b/lib/KorAP/XML/Annotation/CoreNLP/NamedEntities.pm
index 0a9862e..c676564 100644
--- a/lib/KorAP/XML/Annotation/CoreNLP/NamedEntities.pm
+++ b/lib/KorAP/XML/Annotation/CoreNLP/NamedEntities.pm
@@ -24,9 +24,7 @@
             ($found = $found->{f}) &&
             ($found->{-name} eq 'ent') &&
             ($found = $found->{'#text'})) {
-        $mtt->add(
-          term => 'corenlp/ne:' . $found
-        );
+        $mtt->add_by_term('corenlp/ne:' . $found);
       };
     }) or return;
 
diff --git a/lib/KorAP/XML/Annotation/CoreNLP/Sentences.pm b/lib/KorAP/XML/Annotation/CoreNLP/Sentences.pm
index e6b4cc0..37fb19a 100644
--- a/lib/KorAP/XML/Annotation/CoreNLP/Sentences.pm
+++ b/lib/KorAP/XML/Annotation/CoreNLP/Sentences.pm
@@ -11,14 +11,12 @@
     cb => sub {
       my ($stream, $span) = @_;
       my $mtt = $stream->pos($span->get_p_start);
-      $mtt->add(
-        term => '<>:corenlp/s:s',
-        o_start => $span->get_o_start,
-        o_end => $span->get_o_end,
-        p_end => $span->get_p_end,
-        pti => 64,
-        payload => '<b>0' # Could also be 2 for t/p/s
-      );
+      my $mt = $mtt->add_by_term('<>:corenlp/s:s');
+      $mt->set_o_start($span->get_o_start);
+      $mt->set_o_end($span->get_o_end);
+      $mt->set_p_end($span->get_p_end);
+      $mt->set_pti(64);
+      $mt->set_payload('<b>0'); # Could also be 2 for t/p/s
       $i++;
     }
   ) or return;
diff --git a/lib/KorAP/XML/Annotation/DGD/Morpho.pm b/lib/KorAP/XML/Annotation/DGD/Morpho.pm
index aacf00e..c7ade52 100644
--- a/lib/KorAP/XML/Annotation/DGD/Morpho.pm
+++ b/lib/KorAP/XML/Annotation/DGD/Morpho.pm
@@ -32,32 +32,32 @@
         my $name = $feat->{-name};
 
         if (my $t = $conv{$name}) {
-          $mtt->add('dgd/' . $t . ':' . $text);
+          $mtt->add_by_term('dgd/' . $t . ':' . $text);
         }
 
         # Pause
         elsif ($name eq 'pause') {
-          my $p = $mtt->add('dgd/para:pause');
+          my $p = $mtt->add_by_term('dgd/para:pause');
           $p->set_pti(128);
           $p->set_payload('<s>' . $tui);
 
           # Duration
           if ($text =~ /dur="PT([^"]+?)"/) {
-            $p = $mtt->add('@:dgd/para:dur:' . $1);
+            $p = $mtt->add_by_term('@:dgd/para:dur:' . $1);
             $p->set_pti(16);
             $p->set_payload('<s>' . $tui);
           };
 
           # Rendering
           if ($text =~ /rend="([^"]+?)"/) {
-            $p = $mtt->add('@:dgd/para:rend:' . $1);
+            $p = $mtt->add_by_term('@:dgd/para:rend:' . $1);
             $p->set_pti(16);
             $p->set_payload('<s>' . $tui);
           };
 
           # Type
           if ($text =~ /type="([^"]+?)"/) {
-            $p = $mtt->add('@:dgd/para:type:' . $1);
+            $p = $mtt->add_by_term('@:dgd/para:type:' . $1);
             $p->set_pti(16);
             $p->set_payload('<s>' . $tui);
           };
@@ -67,20 +67,20 @@
 
         # Incident
         elsif (($name eq 'incident') || ($name eq 'vocal')) {
-          my $i = $mtt->add('dgd/para:' . $name);
+          my $i = $mtt->add_by_term('dgd/para:' . $name);
           $i->set_pti(128);
           $i->set_payload('<s>' . $tui);
 
           # Rendering
           if ($text =~ /rend="([^"]+?)"/) {
-            $i = $mtt->add('@:dgd/para:rend:' . $1);
+            $i = $mtt->add_by_term('@:dgd/para:rend:' . $1);
             $i->set_pti(16);
             $i->set_payload('<s>' . $tui);
           };
 
           # desc
           if ($text =~ m!<desc[^>]*>([^<]+?)<\/desc>!) {
-            $i = $mtt->add('@:dgd/para:desc:' . $1);
+            $i = $mtt->add_by_term('@:dgd/para:desc:' . $1);
             $i->set_pti(16);
             $i->set_payload('<s>' . $tui);
           };
diff --git a/lib/KorAP/XML/Annotation/DGD/Structure.pm b/lib/KorAP/XML/Annotation/DGD/Structure.pm
index 444bf7e..ce4a25c 100644
--- a/lib/KorAP/XML/Annotation/DGD/Structure.pm
+++ b/lib/KorAP/XML/Annotation/DGD/Structure.pm
@@ -68,7 +68,7 @@
     my $mtt = $stream->pos($p_start);
 
     # Add the base sentence
-    my $mt = $mtt->add('<>:base/s:s');
+    my $mt = $mtt->add_by_term('<>:base/s:s');
     $mt->set_o_start($o_start);
     $mt->set_o_end($_->[1]);
     $mt->set_p_start($p_start);
diff --git a/lib/KorAP/XML/Annotation/DRuKoLa/Morpho.pm b/lib/KorAP/XML/Annotation/DRuKoLa/Morpho.pm
index 5fa9204..3cdfe8b 100644
--- a/lib/KorAP/XML/Annotation/DRuKoLa/Morpho.pm
+++ b/lib/KorAP/XML/Annotation/DRuKoLa/Morpho.pm
@@ -20,7 +20,7 @@
         # pos tag
         if (($f->{-name} eq 'pos') &&
               ($found = $f->{'#text'})) {
-          $mtt->add('drukola/p:' . $found);
+          $mtt->add_by_term('drukola/p:' . $found);
         }
 
         # ana tag
@@ -28,7 +28,7 @@
                  ($found = $f->{'#text'})) {
           my ($pos, $msd) = split(/ /, $found);
           if ($msd) {
-            $mtt->add('drukola/p:' . $pos);
+            $mtt->add_by_term('drukola/p:' . $pos);
           }
           else {
             $msd = $pos;
@@ -38,7 +38,7 @@
           foreach (split '\|', $msd) {
             my ($x, $y) = split "=", $_;
             # case, tense, number, mood, person, degree, gender
-            $mtt->add('drukola/m:' . $x . ($y ? ':' . $y : ''));
+            $mtt->add_by_term('drukola/m:' . $x . ($y ? ':' . $y : ''));
           };
         }
 
@@ -47,7 +47,7 @@
                  && ($found = $f->{'#text'})
                  && $found ne '--') {
           # b($found)->decode('latin-1')->encode->to_string
-          $mtt->add('drukola/l:' . $found);
+          $mtt->add_by_term('drukola/l:' . $found);
         };
       };
     }) or return;
diff --git a/lib/KorAP/XML/Annotation/DeReKo/Structure.pm b/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
index 3ac1930..9742437 100644
--- a/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
+++ b/lib/KorAP/XML/Annotation/DeReKo/Structure.pm
@@ -49,7 +49,7 @@
       my $p_end = $span->get_p_end;
 
       # Add structure
-      my $mt = $mtt->add('<>:dereko/s:' . $name);
+      my $mt = $mtt->add_by_term('<>:dereko/s:' . $name);
       $mt->set_o_start($span->get_o_start);
       $mt->set_o_end($span->get_o_end);
       $mt->set_p_start($p_start);
@@ -75,7 +75,7 @@
           $sentences++;
 
           # Add to stream
-          $mtt->add($mt2);
+          $mtt->add_blessed($mt2);
         }
         elsif ($name eq 'p' && index($as_base, 'paragraphs') >= 0) {
           # Clone Multiterm
@@ -85,14 +85,14 @@
           $paragraphs++;
 
           # Add to stream
-          $mtt->add($mt2);
+          $mtt->add_blessed($mt2);
         }
 
         # Add pagebreaks
         elsif ($name eq 'pb' && index($as_base, 'pagebreaks') >= 0) {
           if (my $nr = first { $_->{-name} eq 'n' } @$attrs) {
             if (($nr = $nr->{'#text'}) && looks_like_number($nr)) {
-              my $mt2 = $mtt->add('~:base/s:pb');
+              my $mt2 = $mtt->add_by_term('~:base/s:pb');
               $mt2->set_payload('<i>' . $nr . '<i>' . $span->get_o_start);
               $mt2->set_stored_offsets(0);
             };
@@ -107,7 +107,7 @@
         foreach (@$attrs) {
 
           # Add attributes
-          my $mt = $mtt->add('@:dereko/s:' . $_->{'-name'} . ($_->{'#text'} ? ':' . $_->{'#text'} : ''));
+          my $mt = $mtt->add_by_term('@:dereko/s:' . $_->{'-name'} . ($_->{'#text'} ? ':' . $_->{'#text'} : ''));
           $mt->set_p_start($p_start);
           $mt->set_pti(17);
           $mt->set_payload('<s>' . $tui .($span->get_milestone ? '' : '<i>' . $p_end));
diff --git a/lib/KorAP/XML/Annotation/Malt/Dependency.pm b/lib/KorAP/XML/Annotation/Malt/Dependency.pm
index 8ea01a5..323c776 100644
--- a/lib/KorAP/XML/Annotation/Malt/Dependency.pm
+++ b/lib/KorAP/XML/Annotation/Malt/Dependency.pm
@@ -26,6 +26,8 @@
       my $rel = $content->{rel};
       $rel = [$rel] unless ref $rel eq 'ARRAY';
 
+      my $mt;
+
       # Iterate over relations
       foreach (@$rel) {
         my $label = $_->{-label};
@@ -41,34 +43,31 @@
         if ($target) {
 
           # Unary means, it refers to itself!
-          $mtt->add(
-            term => '>:malt/d:' . $label,
-            pti => 32, # term-to-term relation
-            payload =>
-              '<i>' . $target->get_pos # . # right part token position
+          $mt = $mtt->add_by_term('>:malt/d:' . $label);
+          $mt->set_pti(32); # term-to-term relation
+          $mt->set_payload(
+            '<i>' . $target->get_pos # . # right part token position
               # '<s>0' . # $source_term->tui . # left part tui
               # '<s>0' # . $target_term->tui # right part tui
-            );
+          );
 
           my $target_mtt = $stream->pos($target->get_pos);
 
-          $target_mtt->add(
-            term => '<:malt/d:' . $label,
-            pti => 32, # term-to-term relation
-            payload =>
-              '<i>' . $source->get_pos # . # left part token position
+          $mt = $target_mtt->add_by_term('<:malt/d:' . $label);
+          $mt->set_pti(32); # term-to-term relation
+          $mt->set_payload(
+            '<i>' . $source->get_pos # . # left part token position
               # '<s>0' . # $source_term->tui . # left part tui
               # '<s>0' # . $target_term->tui # right part tui
-            );
+          );
         }
 
         # Relation is possibly term-to-element with a found target!
         elsif ($target = $tokens->span($from, $to)) {
-          $mtt->add(
-            term => '>:malt/d:' . $label,
-            pti => 33, # term-to-element relation
-            payload =>
-              '<i>' . $target->get_o_start . # end position
+          $mt = $mtt->add_by_term('>:malt/d:' . $label);
+          $mt->set_pti(33); # term-to-element relation
+          $mt->set_payload(
+            '<i>' . $target->get_o_start . # end position
               '<i>' . $target->get_o_end . # end position
               '<i>' . $target->get_p_start . # right part start position
               '<i>' . $target->get_p_end # . # right part end position
@@ -77,11 +76,10 @@
             );
 
           my $target_mtt = $stream->pos($target->get_p_start);
-          $target_mtt->add(
-            term => '<:malt/d:' . $label,
-            pti => 34, # element-to-term relation
-            payload =>
-              '<i>' . $target->get_o_start . # end position
+          $mt = $target_mtt->add_by_term('<:malt/d:' . $label);
+          $mt->set_pti(34); # element-to-term relation
+          $mt->set_payload(
+            '<i>' . $target->get_o_start . # end position
               '<i>' . $target->get_o_end . # end position
               '<i>' . $target->get_p_end . # right part end position
               '<i>' . $source->get_pos # . # left part token position
diff --git a/lib/KorAP/XML/Annotation/RWK/Morpho.pm b/lib/KorAP/XML/Annotation/RWK/Morpho.pm
index 8d9d822..62ee40a 100644
--- a/lib/KorAP/XML/Annotation/RWK/Morpho.pm
+++ b/lib/KorAP/XML/Annotation/RWK/Morpho.pm
@@ -22,26 +22,26 @@
         # pos tag
         if (($name eq 'pos') &&
               ($found = $f->{'#text'})) {
-          $mtt->add(term => 'rwk/p:' . $found);
+          $mtt->add_by_term('rwk/p:' . $found);
         }
 
         # normtok tag
         elsif (($name eq 'normtok') &&
               ($found = $f->{'#text'})) {
-          $mtt->add(term => 'rwk/norm:' . $found);
+          $mtt->add_by_term('rwk/norm:' . $found);
         }
 
         # lemma tag
         elsif (($name eq 'lemma')
                  && ($found = $f->{'#text'})
                  && $found ne '--') {
-          $mtt->add(term => 'rwk/l:' . $found);
+          $mtt->add_by_term('rwk/l:' . $found);
         }
 
         # ana tag
         elsif ($name =~ m/^(?:bc|(?:sub)?type|usage|person|pos|case|number|gender|tense|mood|degree)$/ &&
                  ($found = $f->{'#text'})) {
-          $mtt->add(term => 'rwk/m:' . $name . ':' . $found);
+          $mtt->add_by_term('rwk/m:' . $name . ':' . $found);
         };
       };
     }) or return;
diff --git a/lib/KorAP/XML/Annotation/TreeTagger/Morpho.pm b/lib/KorAP/XML/Annotation/TreeTagger/Morpho.pm
index 9215595..110742c 100644
--- a/lib/KorAP/XML/Annotation/TreeTagger/Morpho.pm
+++ b/lib/KorAP/XML/Annotation/TreeTagger/Morpho.pm
@@ -54,27 +54,24 @@
       };
 
       my %term;
+      my $mt;
       foreach (keys %lemma) {
         if ($lemma{$_} < 1) {
-          $mtt->add(
-            term => 'tt/l:' . $_,
-            pti => 129,
-            payload => '<b>' . floor(($lemma{$_} * 255))
-          );
+          $mt = $mtt->add_by_term('tt/l:' . $_);
+          $mt->set_pti(129);
+          $mt->set_payload('<b>' . floor(($lemma{$_} * 255)));
         } else {
-          $mtt->add(term => 'tt/l:' . $_);
+          $mtt->add_by_term('tt/l:' . $_);
         };
       };
 
       foreach (keys %pos) {
         if ($pos{$_} < 1) {
-          $mtt->add(
-            term => 'tt/p:' . $_,
-            pti => 129,
-            payload => '<b>' . floor(($pos{$_} * 255))
-          );
+          $mt = $mtt->add_by_term('tt/p:' . $_);
+          $mt->set_pti(129);
+          $mt->set_payload('<b>' . floor(($pos{$_} * 255)));
         } else {
-          $mtt->add(term => 'tt/p:' . $_);
+          $mtt->add_by_term('tt/p:' . $_);
         };
       };
 
diff --git a/lib/KorAP/XML/Annotation/TreeTagger/Sentences.pm b/lib/KorAP/XML/Annotation/TreeTagger/Sentences.pm
index 567dace..bdfcbc5 100644
--- a/lib/KorAP/XML/Annotation/TreeTagger/Sentences.pm
+++ b/lib/KorAP/XML/Annotation/TreeTagger/Sentences.pm
@@ -11,14 +11,12 @@
     cb => sub {
       my ($stream, $span) = @_;
       my $mtt = $stream->pos($span->get_p_start);
-      $mtt->add(
-        term => '<>:tt/s:s',
-        o_start => $span->get_o_start,
-        o_end => $span->get_o_end,
-        p_end => $span->get_p_end,
-        pti => 64,
-        payload => '<b>0' # Could be 2 as well t/p/s
-      );
+      my $mt = $mtt->add_by_term('<>:tt/s:s');
+      $mt->set_o_start($span->get_o_start);
+      $mt->set_o_end($span->get_o_end);
+      $mt->set_p_end($span->get_p_end);
+      $mt->set_pti(64);
+      $mt->set_payload('<b>0'); # Could be 2 as well t/p/s
       $i++;
     }
   ) or return;
diff --git a/lib/KorAP/XML/Index/MultiTerm.pm b/lib/KorAP/XML/Index/MultiTerm.pm
index 863295f..85dd0aa 100644
--- a/lib/KorAP/XML/Index/MultiTerm.pm
+++ b/lib/KorAP/XML/Index/MultiTerm.pm
@@ -7,19 +7,22 @@
 # Everything should be stored as bytes already (if this is feasible)
 
 use constant {
-  PAYLOAD => 0,
-  P_START => 1,
-  P_END => 2,
-  O_START => 3,
-  O_END => 4,
-  TERM => 5,
-  STORED_OFFSETS => 6,
-  PTI => 7, # former 10
-  TUI => 8,
+  TERM           => 0,
+  O_START        => 1,
+  O_END          => 2,
+  P_START        => 3,
+  P_END          => 4,
+  STORED_OFFSETS => 5,
+  PTI            => 6,
+  TUI            => 7,
+  PAYLOAD        => 8,
 };
 
 sub new {
   my $self = bless [], shift;
+
+  # TODO:
+  #   Deprecate!
   for (my $i = 0; $i < scalar @_; $i+=2) {
     if ($_[$i] eq 'term') {
       $self->[TERM] = $_[$i+1];
@@ -52,8 +55,8 @@
   $self;
 };
 
-sub new_from_array {
-  bless [@_], shift;
+sub new_from_term {
+  bless [$_[1]], $_[0];
 };
 
 sub new_blank {
diff --git a/lib/KorAP/XML/Index/MultiTermToken.pm b/lib/KorAP/XML/Index/MultiTermToken.pm
index ef9d568..3a2745b 100644
--- a/lib/KorAP/XML/Index/MultiTermToken.pm
+++ b/lib/KorAP/XML/Index/MultiTermToken.pm
@@ -42,16 +42,16 @@
   $mt;
 };
 
-sub add_position_term {
-  my $self = shift;
-  my $mt = KorAP::XML::Index::MultiTerm->new_blank;
-  $mt->set_term('_'. $_[0]);
-  $mt->set_o_start($_[1]);
-  $mt->set_o_end($_[2]);
-  push(@{$self->[MT]}, $mt);
+sub add_by_term {
+  my $mt = KorAP::XML::Index::MultiTerm->new_from_term($_[1]);
+  push(@{$_[0]->[MT]}, $mt);
   $mt;
 };
 
+sub add_blessed {
+  push(@{$_[0]->[MT]}, $_[1]);
+  $_[1];
+};
 
 sub set_o_start {
   return $_[0]->[O_START] = $_[1];
diff --git a/lib/KorAP/XML/Index/MultiTermTokenStream.pm b/lib/KorAP/XML/Index/MultiTermTokenStream.pm
index 456f5f6..0ec97c7 100644
--- a/lib/KorAP/XML/Index/MultiTermTokenStream.pm
+++ b/lib/KorAP/XML/Index/MultiTermTokenStream.pm
@@ -32,23 +32,11 @@
     return $node if $node;
 
     my $tui = $self->tui($unit->get_pos);
-    #    return $mtt->add(
-    #      term => $term,
-    #      pti => 128,
-    #      payload => '<s>' . $tui,
-    #      tui => $tui
-    #    );
-    return $mtt->add_as_array(
-      '<s>' . $tui, # PAYLOAD=0
-      undef,
-      undef,
-      undef,
-      undef,
-      $term,        # TERM=5
-      undef,
-      128,         # PTI=7
-      $tui          # TUI=8
-    )
+    my $mt = $mtt->add_by_term($term);
+    $mt->set_payload('<s>' . $tui);
+    $mt->set_pti(128);
+    $mt->set_tui($tui);
+    return $mt;
   }
 
   # Is span
diff --git a/lib/KorAP/XML/Tokenizer.pm b/lib/KorAP/XML/Tokenizer.pm
index 51978cd..ad38f20 100644
--- a/lib/KorAP/XML/Tokenizer.pm
+++ b/lib/KorAP/XML/Tokenizer.pm
@@ -174,10 +174,10 @@
 
     # Add surface term
     # That's always the first term!
-    $mtt->add('s:' . $token);
+    $mtt->add_by_term('s:' . $token);
 
     # Add case insensitive term
-    $mtt->add('i:' . fc $token);
+    $mtt->add_by_term('i:' . fc $token);
 
     # Add offset information
     $mtt->set_o_start($from);
@@ -190,11 +190,9 @@
     $old = $to + 1;
 
     # Add position term
-    $mtt->add_position_term(
-      $have,
-      $mtt->get_o_start,
-      $mtt->get_o_end
-    );
+    my $mt = $mtt->add_by_term('_' . $have);
+    $mt->set_o_start($mtt->get_o_start);
+    $mt->set_o_end($mtt->get_o_end);
 
     $have++;
   };
@@ -208,16 +206,16 @@
   $mtts->add_meta('tokens', '<i>' . $have);
 
   # Add text boundary
-  my $tb = $mtts->pos(0)->add('<>:base/s:t');
+  my $tb = $mtts->pos(0)->add_by_term('<>:base/s:t');
   $tb->set_o_start(0);
   $tb->set_p_end($have);
-  $tb->set_o_end($doc->primary->data_length);
+  $tb->set_o_end($p->data_length);
   $tb->set_payload('<b>0');
   $tb->set_pti(64);
 
   # Create a gap for the end
-  if ($doc->primary->data_length >= ($old - 1)) {
-    $range->gap($old, $doc->primary->data_length + 1, $have)
+  if ($p->data_length >= ($old - 1)) {
+    $range->gap($old, $p->data_length + 1, $have)
   };
 
   # Add info