Added Elephant and Spacy to empirist and ud benchmarks
Change-Id: Ie12f7334f031d3ad6cbf42f4be1268cb8e1cb99f
diff --git a/.gitignore b/.gitignore
index 0e95a24..b044dfd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,6 +2,7 @@
/corpus/de_*-ud-train*
/sandbox
/Sandbox
+/output_*
/corpus/*.txt
\#*
*~
diff --git a/benchmarks/empirist.pl b/benchmarks/empirist.pl
index 24687ec..9c063c7 100644
--- a/benchmarks/empirist.pl
+++ b/benchmarks/empirist.pl
@@ -51,6 +51,14 @@
my $raw = $gold_path . $_[1] . '/raw/' . $_[0];
system 'somajo-tokenizer ' . $raw . ' 2> /dev/null > ' . $empirist_path . $_[1] . '/somajo/' . $_[0];
},
+ elephant => sub {
+ my $raw = $gold_path . $_[1] . '/raw/' . $_[0];
+ system './elephant-wrapper/bin/tokenize.sh -i ' . $raw . ' UD_German | sed "s/\s/\n/g" > ' . $empirist_path . $_[1] . '/elephant/' . $_[0];
+ },
+ spacy => sub {
+ my $raw = $gold_path . $_[1] . '/raw/' . $_[0];
+ system 'python3 ./spacy/spacy_tok.py ' . $raw . ' > ' . $empirist_path . $_[1] . '/spacy/' . $_[0];
+ },
stanford => sub {
my $raw = $gold_path . $_[1] . '/raw/' . $_[0];
system 'CLASSPATH=/euralex/stanford-corenlp-4.4.0/* java edu.stanford.nlp.pipeline.StanfordCoreNLP ' .
@@ -60,20 +68,24 @@
}
);
-#delete $tools{waste};
-#delete $tools{datok};
-#delete $tools{korap_tokenizer};
-#delete $tools{opennlp_simple};
-#delete $tools{opennlp_tokenizer};
-#delete $tools{tree_tagger};
-#delete $tools{jtok};
-#delete $tools{syntok};
-#delete $tools{somajo};
-#delete $tools{stanford};
+# delete $tools{waste};
+# delete $tools{datok};
+# delete $tools{korap_tokenizer};
+# delete $tools{opennlp_simple};
+# delete $tools{opennlp_tokenizer};
+# delete $tools{tree_tagger};
+# delete $tools{jtok};
+# delete $tools{syntok};
+# delete $tools{somajo};
+# delete $tools{stanford};
+# delete $tools{spacy};
+# delete $tools{elephant};
# Create project folders
foreach (keys %tools) {
+ system 'rm -r ' . $empirist_path . 'cmc/' . $_;
mkdir $empirist_path . 'cmc/' . $_;
+ system 'rm -r ' . $empirist_path . 'web/' . $_;
mkdir $empirist_path . 'web/' . $_;
};
diff --git a/benchmarks/ud_tokens.pl b/benchmarks/ud_tokens.pl
index 6ae8fba..089ee51 100644
--- a/benchmarks/ud_tokens.pl
+++ b/benchmarks/ud_tokens.pl
@@ -46,6 +46,12 @@
syntok => sub {
system 'python3 -m syntok.tokenizer ' . $raw . ' | sed "s/\s/\n/g" > ' . $ud_path . '/syntok/' . $base;
},
+ elephant => sub {
+ system './elephant-wrapper/bin/tokenize.sh -i ' . $raw . ' UD_German | sed "s/\s/\n/g" > ' . $ud_path . '/elephant/' . $base;
+ },
+ spacy => sub {
+ system 'python3 ./spacy/spacy_tok.py ' . $raw . ' > ' . $ud_path . '/spacy/' . $base;
+ },
somajo => sub {
system 'somajo-tokenizer ' . $raw . ' 2> /dev/null > ' . $ud_path . '/somajo/' . $base;
},
@@ -57,16 +63,18 @@
}
);
-# delete $tools{waste};
-# delete $tools{datok};
-# delete $tools{korap_tokenizer};
-# delete $tools{opennlp_simple};
-# delete $tools{opennlp_tokenizer};
-# delete $tools{tree_tagger};
-# delete $tools{jtok};
-# delete $tools{syntok};
-# delete $tools{somajo};
-# delete $tools{stanford};
+ delete $tools{waste};
+ delete $tools{datok};
+ delete $tools{korap_tokenizer};
+ delete $tools{opennlp_simple};
+ delete $tools{opennlp_tokenizer};
+ delete $tools{tree_tagger};
+ delete $tools{jtok};
+ delete $tools{syntok};
+ delete $tools{somajo};
+ delete $tools{stanford};
+# delete $tools{elephant};
+# delete $tools{spacy};
# Create project folders
foreach (keys %tools) {