Add eos evaluation
Change-Id: Ia721ce1df8798fa2771059b4feb12eb56459325b
diff --git a/benchmarks/cleanup/split_conllu.pl b/benchmarks/cleanup/split_conllu.pl
index 9dfd824..441be36 100644
--- a/benchmarks/cleanup/split_conllu.pl
+++ b/benchmarks/cleanup/split_conllu.pl
@@ -9,6 +9,7 @@
open(X, '<' . $file);
open(RAW, '>' . $file . '.raw');
open(SPLIT, '>' . $file . '.split');
+open(EOS, '>' . $file . '.eos');
my $init;
@@ -21,6 +22,9 @@
print RAW ' ';
};
print RAW $1;
+ my $temp = $1;
+ $temp =~ s/[\s\n\t]+//g;
+ print EOS $temp, "\n";
}
elsif (m/^\d+[\s\t]/) {
if (/^\d+[\s\t]+([^\t\s]+)[\t\s]/) {
@@ -32,4 +36,5 @@
close(X);
close(RAW);
+close(EOS);
close(SPLIT);