Support koral versioning
Change-Id: Id07e878843c790c55aca0e2da0869f4cd2e9150c
diff --git a/t/tokenization.t b/t/tokenization.t
index 4e883d1..33b0911 100644
--- a/t/tokenization.t
+++ b/t/tokenization.t
@@ -4,6 +4,7 @@
use utf8;
use Test::More;
use JSON::XS;
+use Data::Dumper;
use File::Basename 'dirname';
use File::Spec::Functions 'catdir';
@@ -70,6 +71,10 @@
is($json->{fields}->[6]->{key}, 'creationDate');
is($json->{fields}->[6]->{value}, '2005');
+is($json->{data}->{name}, 'tokens');
+is($json->{data}->{tokenSource}, 'opennlp#tokens');
+is($json->{data}->{stream}->[0]->[1], '<>:base/s:t$<b>64<i>0<i>6083<i>1067<b>0');
+
done_testing;
__END__