commit e8b9dd2fd0b13a84ae1381bffd38e67965fbf79c
author Marc Kupietz <kupietz@ids-mannheim.de> Thu Jan 03 17:55:59 2019 +0100
committer Marc Kupietz <kupietz@ids-mannheim.de> Thu Jan 03 17:55:59 2019 +0100
tree ed963b6ef19c690cffd4dd452bec51c26072c683
parent 257a93deec19b900f6c97f89f2a5590c10b79e9b

w2vserver: sanitize deduping with merged vecs

w2v-server.pl

diff --git a/w2v-server.pl b/w2v-server.pl
index c442a2f..6f8d7f5 100755
--- a/w2v-server.pl
+++ b/w2v-server.pl
@@ -975,7 +975,7 @@
   for (a = 0, i = 0; i < N && a < N*para_threads; a++) {
     int filtered=0;
     long long c = best[a].wordi;
-    if (dedupe && i > 0) {
+    if ((merge_words && dedupe && i > 1) || (!merge_words && dedupe && i > 0)) {
       for (j=0; j<i && !filtered; j++)
         if (strcasestr(&vocab[c * max_w], &vocab[chosen[j] * max_w]) ||
             strcasestr(&vocab[chosen[j] * max_w], &vocab[c * max_w])) {