commit 91728fe4df3cb8a292467c8081f66f7f667d27ed
author Nils Diewald <nils@diewald-online.de> Tue Nov 04 02:21:05 2014 +0000
committer Nils Diewald <nils@diewald-online.de> Tue Nov 04 02:21:05 2014 +0000
tree 17069a062e7c2ab35d6d24d8dc07df383d5aefc0
parent 31ec1286d8ebe4bce15830bb3746eb415754e98a

Minor fix for broken offsets

lib/KorAP/Tokenizer.pm

diff --git a/lib/KorAP/Tokenizer.pm b/lib/KorAP/Tokenizer.pm
index cbaae8c..9ff8c51 100644
--- a/lib/KorAP/Tokenizer.pm
+++ b/lib/KorAP/Tokenizer.pm
@@ -279,7 +279,8 @@
 
   if ($cb) {
     foreach (@$spanarray) {
-      $cb->($self->stream, $_); #, $spans);
+      $cb->($self->stream, $_) if defined $_->p_start;
+      #, $spans);
     };
     return 1;
   };
@@ -325,7 +326,8 @@
 
   if ($cb) {
     foreach (@$tokenarray) {
-      $cb->($self->stream, $_); #, $tokens);
+      $cb->($self->stream, $_) if defined $_->pos;
+      #, $tokens);
     };
     return 1;
   };