Add eos evaluation
Change-Id: Ia721ce1df8798fa2771059b4feb12eb56459325b
diff --git a/benchmarks/cleanup/jtok.pl b/benchmarks/cleanup/jtok.pl
new file mode 100644
index 0000000..664e919
--- /dev/null
+++ b/benchmarks/cleanup/jtok.pl
@@ -0,0 +1,22 @@
+#!/usr/bin/env perl
+use strict;
+use warnings;
+
+my $init = 1;
+my $c = '';
+foreach (<>) {
+ if (s/\s +Token: \"// && s/^(\"?[^\"]*?)\".+?$/$1/g) {
+ $c .= $_;
+ }
+ elsif (m/Text Unit Start/) {
+ if ($init) {
+ $init = 0;
+ } else {
+ $c =~ s/[\s\n\t]+//g;
+ print $c,"\n";
+ $c = '';
+ };
+ };
+};
+
+print "\n";