blob: 27682962f2d40cab9b35995d182dac213762aa03 [file] [log] [blame]
Marc Kupietzdc22b982015-10-09 09:19:34 +02001#!/usr/local/bin/perl
2use Inline C;
Marc Kupietzdb2dc7e2017-12-02 12:04:03 +01003use Inline C => Config => CLEAN_AFTER_BUILD => 0, ccflags => $Config{ccflags}." -Wall -O4";
Marc Kupietza5f60042017-05-04 10:38:12 +02004#use Inline C => Config => BUILD_NOISY => 1, CFLAGS => $Config{cflags}." -O4 -mtune k9";
5#use Inline C => Config => CLEAN_AFTER_BUILD => 0, ccflags => $Config{ccflags}." -Ofast -march k8 -mtune k8 ";
Marc Kupietzdc22b982015-10-09 09:19:34 +02006use Mojolicious::Lite;
Marc Kupietzc4893362016-02-25 08:04:46 +01007use Mojo::JSON qw(decode_json encode_json to_json);
Marc Kupietz30ca4342017-11-22 21:21:20 +01008use base 'Mojolicious::Plugin';
9
Marc Kupietz247500f2015-10-09 11:29:01 +020010use Encode qw(decode encode);
Marc Kupietza5b90152016-03-15 17:39:19 +010011use Getopt::Std;
Marc Kupietz7bc85fd2016-02-24 11:42:41 +010012use Mojo::Server::Daemon;
Marc Kupietzffef9302017-11-07 15:58:01 +010013use Cwd;
14app->static->paths->[0] = getcwd;
15
Marc Kupietzd4227392016-03-01 16:45:12 +010016plugin 'Log::Access';
Marc Kupietzb3422c12017-07-04 14:12:11 +020017plugin "RequestBase";
Marc Kupietzdc22b982015-10-09 09:19:34 +020018
Marc Kupietza5b90152016-03-15 17:39:19 +010019our $opt_i = 0; # latin1-input?
20our $opt_l = undef;
21our $opt_p = 5676;
Marc Kupietza2e64502016-04-27 09:53:51 +020022our $opt_m;
Marc Kupietz6ed81872016-04-27 14:04:04 +020023our $opt_M;
Marc Kupietz43ee87e2016-04-25 10:50:08 +020024our $opt_n = '';
25our $opt_d;
Marc Kupietz5c3887d2016-04-28 08:53:35 +020026our $opt_G;
Marc Kupietza5b90152016-03-15 17:39:19 +010027
Marc Kupietz6ed81872016-04-27 14:04:04 +020028my %marked;
Marc Kupietz793413b2016-04-02 21:48:57 +020029my $training_args="";
Marc Kupietza2e64502016-04-27 09:53:51 +020030my $mergedEnd=0;
Marc Kupietz15987412017-11-07 15:56:58 +010031my %cache;
Marc Kupietz793413b2016-04-02 21:48:57 +020032
Marc Kupietza5f60042017-05-04 10:38:12 +020033getopts('d:Gil:p:m:n:M:');
Marc Kupietz6ed81872016-04-27 14:04:04 +020034
35if($opt_M) {
Marc Kupietzed930212016-04-27 15:42:38 +020036 open my $handle, '<:encoding(UTF-8)', $opt_M
37 or die "Can't open '$opt_M' for reading: $!";
38 while(<$handle>) {
Marc Kupietz6ed81872016-04-27 14:04:04 +020039 foreach my $mw (split /\s+/) {
40 $marked{$mw}=1
41 }
42 }
Marc Kupietzed930212016-04-27 15:42:38 +020043 close($handle);
Marc Kupietz6ed81872016-04-27 14:04:04 +020044}
Marc Kupietza5b90152016-03-15 17:39:19 +010045
Marc Kupietz7bc85fd2016-02-24 11:42:41 +010046# -cbow 1 -size 200 -window 8 -negative 25 -hs 0 -sample 1e-4 -threads 40 -binary 1 -iter 15
Marc Kupietza5b90152016-03-15 17:39:19 +010047if(!$ARGV[0]) {
Marc Kupietz6b2975c2016-03-18 21:59:33 +010048 init_net("vectors15.bin", $opt_n, ($opt_i? 1 : 0));
Marc Kupietz2cb667e2016-03-10 09:44:12 +010049} else {
Marc Kupietz6b2975c2016-03-18 21:59:33 +010050 init_net($ARGV[0], $opt_n, ($opt_i? 1 : 0));
Marc Kupietz793413b2016-04-02 21:48:57 +020051 if(open(FILE, "$ARGV[0].args")) {
52 $training_args = <FILE>;
53 }
54 close(FILE);
Marc Kupietz2cb667e2016-03-10 09:44:12 +010055}
Marc Kupietzdc22b982015-10-09 09:19:34 +020056
Marc Kupietza2e64502016-04-27 09:53:51 +020057if($opt_m) {
58 $mergedEnd = mergeVectors($opt_m);
59}
60
Marc Kupietz6ed81872016-04-27 14:04:04 +020061
Marc Kupietz43ee87e2016-04-25 10:50:08 +020062if($opt_d) { # -d: dump vecs and exit
63 dump_vecs($opt_d);
64 exit;
65}
66
Marc Kupietza5b90152016-03-15 17:39:19 +010067my $daemon = Mojo::Server::Daemon->new(
68 app => app,
69 listen => ['http://'.($opt_l ? $opt_l : '*').":$opt_p"]
70);
71
Marc Kupietz5c3887d2016-04-28 08:53:35 +020072if($opt_G) {
73 print "Filtering garbage\n";
74 filter_garbage();
75}
76
Marc Kupietz554aff52017-11-09 14:42:09 +010077get '*/js/*' => sub {
Marc Kupietzffef9302017-11-07 15:58:01 +010078 my $c = shift;
79 my $url = $c->req->url;
80 $url =~ s@/derekovecs@@g;
81 $c->app->log->info("GET: " . $url);
82 $c->reply->static($url);
83};
84
Marc Kupietzdf3d4b52017-11-29 16:57:27 +010085get '*/img/*' => sub {
86 my $c = shift;
87 my $url = $c->req->url;
88 $url =~ s@/derekovecs@@g;
89 $c->app->log->info("GET: " . $url);
90 $c->reply->static($url);
91};
92
Marc Kupietzdc22b982015-10-09 09:19:34 +020093get '/' => sub {
94 my $c = shift;
Marc Kupietza5f60042017-05-04 10:38:12 +020095 $c->app->log->info("get: ".$c->req->url->to_abs);
Marc Kupietzdc22b982015-10-09 09:19:34 +020096 my $word=$c->param('word');
Marc Kupietz44bee3c2016-02-25 16:26:29 +010097 my $no_nbs=$c->param('n') || 100;
98 my $no_iterations=$c->param('N') || 2000;
Marc Kupietzd4227392016-03-01 16:45:12 +010099 my $perplexity=$c->param('perplexity') || 20;
Marc Kupietzc4d62f82016-03-01 11:04:24 +0100100 my $epsilon=$c->param('epsilon') || 5;
Marc Kupietzd7aea722016-03-02 11:59:12 +0100101 my $som=$c->param('som') || 0;
Marc Kupietza2e64502016-04-27 09:53:51 +0200102 my $searchBaseVocabFirst=$c->param('sbf') || 0;
Marc Kupietz6d9a6782016-03-23 17:25:25 +0100103 my $sort=$c->param('sort') || 0;
Marc Kupietzc469f3b2017-11-13 14:07:36 +0100104 my $csv=$c->param('csv') || 0;
Marc Kupietzb613b052016-04-28 14:11:59 +0200105 my $json=$c->param('json') || 0;
Marc Kupietzdb2dc7e2017-12-02 12:04:03 +0100106 my $cutoff=$c->param('cutoff') || 500000;
Marc Kupietzd91212f2017-11-13 10:05:09 +0100107 my $dedupe=$c->param('dedupe') || 0;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100108 my $res;
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100109 my @lists;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100110 my @collocations;
Marc Kupietz7bc85fd2016-02-24 11:42:41 +0100111 if(defined($word) && $word !~ /^\s*$/) {
112 $c->inactivity_timeout(300);
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100113 $word =~ s/\s+/ /g;
114 for my $w (split(' *\| *', $word)) {
Marc Kupietzd91212f2017-11-13 10:05:09 +0100115 if ($cache{$w.$cutoff.$no_nbs.$sort.$dedupe}) {
Marc Kupietz15987412017-11-07 15:56:58 +0100116 $c->app->log->info("Getting $w results from cache");
Marc Kupietzd91212f2017-11-13 10:05:09 +0100117 $res = $cache{$w.$cutoff.$no_nbs.$sort.$dedupe}
Marc Kupietza5b90152016-03-15 17:39:19 +0100118 } else {
Marc Kupietz15987412017-11-07 15:56:58 +0100119 $c->app->log->info('Looking for neighbours of '.$w);
120 if($opt_i) {
Marc Kupietzd91212f2017-11-13 10:05:09 +0100121 $res = get_neighbours(encode("iso-8859-1", $w), $no_nbs, $sort, $searchBaseVocabFirst, $cutoff, $dedupe);
Marc Kupietz15987412017-11-07 15:56:58 +0100122 } else {
Marc Kupietzd91212f2017-11-13 10:05:09 +0100123 $res = get_neighbours($w, $no_nbs, $sort, $searchBaseVocabFirst, $cutoff, $dedupe);
Marc Kupietz15987412017-11-07 15:56:58 +0100124 }
Marc Kupietz2dd2dd72017-12-01 22:08:14 +0100125 $cache{$w.$cutoff.$no_nbs.$sort.$dedupe} = $res;
Marc Kupietza5b90152016-03-15 17:39:19 +0100126 }
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100127 push(@lists, $res->{paradigmatic});
Marc Kupietz15987412017-11-07 15:56:58 +0100128 }
129 }
Marc Kupietz000ad862016-02-26 14:59:12 +0100130 $word =~ s/ *\| */ | /g;
Marc Kupietzb613b052016-04-28 14:11:59 +0200131 if($json) {
132 return $c->render(json => {word => $word, list => \@lists, collocators=>$res->{syntagmatic}});
Marc Kupietzc469f3b2017-11-13 14:07:36 +0100133 } elsif($csv) {
134 my $csv_data="";
135 for (my $i=0; $i <= $no_nbs; $i++) {
136 $csv_data .= $res->{paradigmatic}->[$i]->{word} . ", ";
137 }
138 for (my $i=0; $i < $no_nbs; $i++) {
139 $csv_data .= $res->{syntagmatic}->[$i]->{word} . ", ";
140 }
141 chop $csv_data;
142 chop $csv_data;
143 $csv_data .= "\n";
144 return $c->render(text=>$csv_data);
Marc Kupietzb613b052016-04-28 14:11:59 +0200145 } else {
Marc Kupietzd91212f2017-11-13 10:05:09 +0100146 $c->render(template=>"index", word=>$word, cutoff=>$cutoff, no_nbs=>$no_nbs, no_iterations => $no_iterations, epsilon=> $epsilon, perplexity=> $perplexity, show_som=>$som, searchBaseVocabFirst=>$searchBaseVocabFirst, sort=>$sort, training_args=>$training_args, mergedEnd=> $mergedEnd, dedupe=> $dedupe, marked=>\%marked, lists=> \@lists, collocators=> $res->{syntagmatic});
Marc Kupietzb613b052016-04-28 14:11:59 +0200147 }
Marc Kupietzdc22b982015-10-09 09:19:34 +0200148};
149
Marc Kupietz30ca4342017-11-22 21:21:20 +0100150helper(bitvec2window => sub {
151 my ($self, $n) = @_;
152 my $str = unpack("B32", pack("N", $n));
153 $str =~ s/^\d{22}//;
154 $str =~ s/^(\d{5})/$1x/;
155 $str =~ s/0/·/g;
156 $str =~ s/1/+/g;
157 return $str;
158 });
159
Marc Kupietza5b90152016-03-15 17:39:19 +0100160$daemon->run; # app->start;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200161
162exit;
163
164__END__
165
166__C__
167#include <stdio.h>
168#include <string.h>
169#include <math.h>
170#include <malloc.h>
171#include <stdlib.h> //strlen
Marc Kupietzf0809762016-02-26 10:13:47 +0100172#include <sys/mman.h>
Marc Kupietz000ad862016-02-26 14:59:12 +0100173#include <pthread.h>
Marc Kupietzdc22b982015-10-09 09:19:34 +0200174
175#define max_size 2000
176#define max_w 50
Marc Kupietz7bc85fd2016-02-24 11:42:41 +0100177#define MAX_NEIGHBOURS 1000
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100178#define MAX_WORDS -1
Marc Kupietz000ad862016-02-26 14:59:12 +0100179#define MAX_THREADS 100
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100180#define MAX_CC 50
181#define EXP_TABLE_SIZE 1000
182#define MAX_EXP 6
Marc Kupietz271e2a42016-03-22 11:37:43 +0100183#define MIN_RESP 0.50
Marc Kupietzdc22b982015-10-09 09:19:34 +0200184
185//the thread function
186void *connection_handler(void *);
Marc Kupietz000ad862016-02-26 14:59:12 +0100187
188typedef struct {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100189 long long wordi;
190 long position;
191 float activation;
Marc Kupietz4116b432017-12-06 14:15:32 +0100192 float average;
Marc Kupietza77acce2017-11-30 16:59:07 +0100193 float cprobability; // column wise probability
Marc Kupietz4116b432017-12-06 14:15:32 +0100194 float cprobability_sum;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100195 float probability;
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100196 float activation_sum;
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100197 float max_activation;
Marc Kupietz4116b432017-12-06 14:15:32 +0100198 float heat[16];
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100199} collocator;
200
201typedef struct {
202 collocator *best;
Marc Kupietz80abb442016-03-23 21:04:08 +0100203 int length;
Marc Kupietz000ad862016-02-26 14:59:12 +0100204} knn;
Marc Kupietz271e2a42016-03-22 11:37:43 +0100205
Marc Kupietz000ad862016-02-26 14:59:12 +0100206typedef struct {
Marc Kupietz48c29682016-03-19 11:30:43 +0100207 long long wordi[MAX_NEIGHBOURS];
208 char sep[MAX_NEIGHBOURS];
209 int length;
210} wordlist;
211
212typedef struct {
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100213 long cutoff;
Marc Kupietz48c29682016-03-19 11:30:43 +0100214 wordlist *wl;
Marc Kupietz000ad862016-02-26 14:59:12 +0100215 char *token;
216 int N;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100217 long from;
Marc Kupietz000ad862016-02-26 14:59:12 +0100218 unsigned long upto;
Marc Kupietzc55c8872017-12-01 23:27:10 +0100219 collocator *best;
Marc Kupietzce3d4c62016-03-23 16:11:25 +0100220 float *target_sums;
Marc Kupietzbd41da32017-11-24 10:26:43 +0100221 float *window_sums;
Marc Kupietz000ad862016-02-26 14:59:12 +0100222} knnpars;
223
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200224float *M, *M2=0L, *syn1neg_window, *expTable;
Marc Kupietzbd41da32017-11-24 10:26:43 +0100225float *window_sums;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200226char *vocab;
Marc Kupietz5c3887d2016-04-28 08:53:35 +0200227char *garbage = NULL;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100228
Marc Kupietza2e64502016-04-27 09:53:51 +0200229long long words, size, merged_end;
Marc Kupietza5f60042017-05-04 10:38:12 +0200230long long merge_words = 0;
Marc Kupietz000ad862016-02-26 14:59:12 +0100231int num_threads=20;
Marc Kupietza5b90152016-03-15 17:39:19 +0100232int latin_enc=0;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100233int window;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200234
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100235int init_net(char *file_name, char *net_name, int latin) {
Marc Kupietz67c20282016-02-26 09:42:00 +0100236 FILE *f, *binvecs, *binwords;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100237 int binwords_fd, binvecs_fd, net_fd, i;
Marc Kupietz82b02672016-02-26 12:32:25 +0100238 long long a, b, c, d, cn;
239 float len;
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200240 double val;
Marc Kupietz82b02672016-02-26 12:32:25 +0100241
Marc Kupietz67c20282016-02-26 09:42:00 +0100242 char binvecs_fname[256], binwords_fname[256];
243 strcpy(binwords_fname, file_name);
244 strcat(binwords_fname, ".words");
245 strcpy(binvecs_fname, file_name);
246 strcat(binvecs_fname, ".vecs");
Marc Kupietzdc22b982015-10-09 09:19:34 +0200247
Marc Kupietza5b90152016-03-15 17:39:19 +0100248 latin_enc = latin;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200249 f = fopen(file_name, "rb");
250 if (f == NULL) {
251 printf("Input file %s not found\n", file_name);
252 return -1;
253 }
254 fscanf(f, "%lld", &words);
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100255 if(MAX_WORDS > 0 && words > MAX_WORDS) words = MAX_WORDS;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200256 fscanf(f, "%lld", &size);
Marc Kupietz2cb667e2016-03-10 09:44:12 +0100257 if( (binvecs_fd = open(binvecs_fname, O_RDONLY)) < 0 || (binwords_fd = open(binwords_fname, O_RDONLY)) < 0) {
258 printf("Converting %s to memory mappable structures\n", file_name);
Marc Kupietzf0809762016-02-26 10:13:47 +0100259 vocab = (char *)malloc((long long)words * max_w * sizeof(char));
260 M = (float *)malloc((long long)words * (long long)size * sizeof(float));
261 if (M == NULL) {
262 printf("Cannot allocate memory: %lld MB %lld %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
263 return -1;
264 }
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200265 if(strstr(file_name, ".txt")) {
266 for (b = 0; b < words; b++) {
267 a = 0;
268 while (1) {
269 vocab[b * max_w + a] = fgetc(f);
270 if (feof(f) || (vocab[b * max_w + a] == ' ')) break;
271 if ((a < max_w) && (vocab[b * max_w + a] != '\n')) a++;
272 }
273 vocab[b * max_w + a] = 0;
274 len = 0;
275 for (a = 0; a < size; a++) {
276 fscanf(f, "%lf", &val);
277 M[a + b * size] = val;
278 len += val * val;
279 }
280 len = sqrt(len);
281 for (a = 0; a < size; a++) M[a + b * size] /= len;
282 }
283 } else {
284 for (b = 0; b < words; b++) {
285 a = 0;
286 while (1) {
287 vocab[b * max_w + a] = fgetc(f);
288 if (feof(f) || (vocab[b * max_w + a] == ' ')) break;
289 if ((a < max_w) && (vocab[b * max_w + a] != '\n')) a++;
290 }
291 vocab[b * max_w + a] = 0;
292 fread(&M[b * size], sizeof(float), size, f);
293 len = 0;
294 for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
295 len = sqrt(len);
296 for (a = 0; a < size; a++) M[a + b * size] /= len;
297 }
298 }
Marc Kupietz67c20282016-02-26 09:42:00 +0100299 if( (binvecs = fopen(binvecs_fname, "wb")) != NULL && (binwords = fopen(binwords_fname, "wb")) != NULL) {
300 fwrite(M, sizeof(float), (long long)words * (long long)size, binvecs);
301 fclose(binvecs);
302 fwrite(vocab, sizeof(char), (long long)words * max_w, binwords);
303 fclose(binwords);
304 }
Marc Kupietz2cb667e2016-03-10 09:44:12 +0100305 }
306 if( (binvecs_fd = open(binvecs_fname, O_RDONLY)) >= 0 && (binwords_fd = open(binwords_fname, O_RDONLY)) >= 0) {
307 M = mmap(0, sizeof(float) * (long long)words * (long long)size, PROT_READ, MAP_SHARED, binvecs_fd, 0);
308 vocab = mmap(0, sizeof(char) * (long long)words * max_w, PROT_READ, MAP_SHARED, binwords_fd, 0);
309 if (M == MAP_FAILED || vocab == MAP_FAILED) {
310 close(binvecs_fd);
311 close(binwords_fd);
312 fprintf(stderr, "Cannot mmap %s or %s\n", binwords_fname, binvecs_fname);
313 exit(-1);
314 }
315 } else {
316 fprintf(stderr, "Cannot open %s or %s\n", binwords_fname, binvecs_fname);
317 exit(-1);
Marc Kupietz67c20282016-02-26 09:42:00 +0100318 }
Marc Kupietzdc22b982015-10-09 09:19:34 +0200319 fclose(f);
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100320
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200321 if(net_name && strlen(net_name) > 0) {
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100322 if( (net_fd = open(net_name, O_RDONLY)) >= 0) {
323 window = (lseek(net_fd, 0, SEEK_END) - sizeof(float) * words * size) / words / size / sizeof(float) / 2;
324 // lseek(net_fd, sizeof(float) * words * size, SEEK_SET);
Marc Kupietz10bec2b2016-03-23 09:41:31 +0100325 // munmap(M, sizeof(float) * words * size);
326 M2 = mmap(0, sizeof(float) * words * size + sizeof(float) * 2 * window * size * words, PROT_READ, MAP_SHARED, net_fd, 0);
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200327 if (M2 == MAP_FAILED) {
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100328 close(net_fd);
329 fprintf(stderr, "Cannot mmap %s\n", net_name);
330 exit(-1);
331 }
Marc Kupietz10bec2b2016-03-23 09:41:31 +0100332 syn1neg_window = M2 + words * size;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100333 } else {
334 fprintf(stderr, "Cannot open %s\n", net_name);
335 exit(-1);
336 }
337 fprintf(stderr, "Successfully memmaped %s. Determined window size: %d\n", net_name, window);
338 }
339
340 expTable = (float *) malloc((EXP_TABLE_SIZE + 1) * sizeof(float));
341 for (i = 0; i < EXP_TABLE_SIZE; i++) {
342 expTable[i] = exp((i / (float) EXP_TABLE_SIZE * 2 - 1) * MAX_EXP); // Precompute the exp() table
343 expTable[i] = expTable[i] / (expTable[i] + 1); // Precompute f(x) = x / (x + 1)
344 }
Marc Kupietzbd41da32017-11-24 10:26:43 +0100345 window_sums = malloc(sizeof(float) * (window+1) * 2);
Marc Kupietzdc22b982015-10-09 09:19:34 +0200346 return 0;
347}
348
Marc Kupietza2e64502016-04-27 09:53:51 +0200349long mergeVectors(char *file_name){
350 FILE *f, *binvecs, *binwords;
351 int binwords_fd, binvecs_fd, net_fd, i;
352 long long a, b, c, d, cn;
353 float len;
354 float *merge_vecs;
355 char *merge_vocab;
Marc Kupietza5f60042017-05-04 10:38:12 +0200356 /* long long merge_words, merge_size; */
357 long long merge_size;
Marc Kupietza2e64502016-04-27 09:53:51 +0200358
359 char binvecs_fname[256], binwords_fname[256];
360 strcpy(binwords_fname, file_name);
361 strcat(binwords_fname, ".words");
362 strcpy(binvecs_fname, file_name);
363 strcat(binvecs_fname, ".vecs");
364
365 f = fopen(file_name, "rb");
366 if (f == NULL) {
367 printf("Input file %s not found\n", file_name);
368 exit -1;
369 }
370 fscanf(f, "%lld", &merge_words);
371 fscanf(f, "%lld", &merge_size);
372 if(merge_size != size){
373 fprintf(stderr, "vectors must have the same length\n");
374 exit(-1);
375 }
376 if( (binvecs_fd = open(binvecs_fname, O_RDONLY)) >= 0 && (binwords_fd = open(binwords_fname, O_RDONLY)) >= 0) {
377 merge_vecs = malloc(sizeof(float) * (words + merge_words) * size);
378 merge_vocab = malloc(sizeof(char) * (words + merge_words) * max_w);
379 if (merge_vecs == NULL || merge_vocab == NULL) {
380 close(binvecs_fd);
381 close(binwords_fd);
382 fprintf(stderr, "Cannot reserve memory for %s or %s\n", binwords_fname, binvecs_fname);
383 exit(-1);
384 }
385 read(binvecs_fd, merge_vecs, merge_words * size * sizeof(float));
386 read(binwords_fd, merge_vocab, merge_words * max_w);
387 } else {
388 fprintf(stderr, "Cannot open %s or %s\n", binwords_fname, binvecs_fname);
389 exit(-1);
390 }
391 printf("Successfully reallocated memory\nMerging...\n");
392 fflush(stdout);
393 memcpy(merge_vecs + merge_words * size, M, words * size * sizeof(float));
394 memcpy(merge_vocab + merge_words * max_w, vocab, words * max_w);
395 munmap(M, words * size * sizeof(float));
396 munmap(vocab, words * max_w);
397 M = merge_vecs;
398 vocab = merge_vocab;
399 merged_end = merge_words;
400 words += merge_words;
401 fclose(f);
402 printf("merged_end: %lld, words: %lld\n", merged_end, words);
403 return((long) merged_end);
404}
405
Marc Kupietz5c3887d2016-04-28 08:53:35 +0200406void filter_garbage() {
407 long i;
Marc Kupietzab591a82016-04-28 14:08:49 +0200408 unsigned char *w, previous, c;
Marc Kupietz5c3887d2016-04-28 08:53:35 +0200409 garbage = malloc(words);
410 memset(garbage, 0, words);
411 for (i = 0; i < words; i++) {
412 w = vocab + i * max_w;
413 previous = 0;
Marc Kupietzab591a82016-04-28 14:08:49 +0200414 while((c = *w++) && !garbage[i]) {
415 if( ((c <= 90 && c >= 65) && (previous >= 97 && previous <= 122)) ||
416 (previous == '-' && (c & 32)) ||
Marc Kupietz33e3aa12017-11-09 16:19:38 +0100417 (previous == 0xc2 && (c == 0xa4 || c == 0xb6 )) ||
418 c == '<'
Marc Kupietzab591a82016-04-28 14:08:49 +0200419 ) {
Marc Kupietz5c3887d2016-04-28 08:53:35 +0200420 garbage[i]=1;
421 continue;
422 }
423 previous = c;
424 }
425 }
426 return;
427}
428
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100429void *getCollocators(void *args) {
430 knnpars *pars = args;
Marc Kupietz271e2a42016-03-22 11:37:43 +0100431 int N = pars->N;
432 int cc = pars->wl->wordi[0];
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100433 knn *nbs = NULL;
434 long window_layer_size = size * window * 2;
435 long a, b, c, d, e, window_offset, target, max_target=0, maxmax_target;
436 float f, max_f, maxmax_f;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100437 float *target_sums, worstbest, wpos_sum;
438 collocator *best;
Marc Kupietzd5642582016-03-19 22:23:13 +0100439
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200440 if(M2 == NULL || cc == -1)
Marc Kupietzd5642582016-03-19 22:23:13 +0100441 return NULL;
442
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100443 a = posix_memalign((void **) &target_sums, 128, pars->cutoff * sizeof(float));
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100444 best = malloc(N * sizeof(collocator));
Marc Kupietz271e2a42016-03-22 11:37:43 +0100445 worstbest = MIN_RESP;
446
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100447 for (b = 0; b < pars->cutoff; b++)
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100448 target_sums[b]=0;
Marc Kupietzb864ccf2016-03-21 22:40:03 +0100449 for (b = 0; b < N; b++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100450 best[b].wordi = -1;
451 best[b].probability = 1;
452 best[b].activation = worstbest;
Marc Kupietzb864ccf2016-03-21 22:40:03 +0100453 }
Marc Kupietz271e2a42016-03-22 11:37:43 +0100454
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100455 d = cc;
456 maxmax_f = -1;
457 maxmax_target = 0;
458
Marc Kupietz271e2a42016-03-22 11:37:43 +0100459 for (a = pars->from; a < pars->upto; a++) {
460 if(a >= window)
461 a++;
Marc Kupietzb864ccf2016-03-21 22:40:03 +0100462 wpos_sum = 0;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100463 printf("window pos: %ld\n", a);
464 if (a != window) {
465 max_f = -1;
466 window_offset = a * size;
467 if (a > window)
468 window_offset -= size;
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100469 for(target = 0; target < pars->cutoff; target ++) {
470 if(garbage && garbage[target]) continue;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100471 if(target == d)
472 continue;
473 f = 0;
474 for (c = 0; c < size; c++)
Marc Kupietz10bec2b2016-03-23 09:41:31 +0100475 f += M2[d* size + c] * syn1neg_window[target * window_layer_size + window_offset + c];
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100476 if (f < -MAX_EXP)
477 continue;
478 else if (f > MAX_EXP)
479 continue;
480 else
481 f = expTable[(int) ((f + MAX_EXP) * (EXP_TABLE_SIZE / MAX_EXP / 2))];
Marc Kupietzb864ccf2016-03-21 22:40:03 +0100482 wpos_sum += f;
Marc Kupietz271e2a42016-03-22 11:37:43 +0100483
Marc Kupietzce3d4c62016-03-23 16:11:25 +0100484 target_sums[target] += f;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100485 if(f > worstbest) {
Marc Kupietz6d9a6782016-03-23 17:25:25 +0100486 for (b = 0; b < N; b++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100487 if (f > best[b].activation) {
488 memmove(best + b + 1, best + b, (N - b -1) * sizeof(collocator));
489 best[b].activation = f;
490 best[b].wordi = target;
491 best[b].position = window-a;
Marc Kupietz33679a32016-03-22 08:49:39 +0100492 break;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100493 }
494 }
Marc Kupietz6d9a6782016-03-23 17:25:25 +0100495 if(b == N - 1)
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100496 worstbest = best[N-1].activation;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100497 }
498 }
499 printf("%d %.2f\n", max_target, max_f);
500 printf("%s (%.2f) ", &vocab[max_target * max_w], max_f);
501 if(max_f > maxmax_f) {
502 maxmax_f = max_f;
503 maxmax_target = max_target;
504 }
Marc Kupietz33679a32016-03-22 08:49:39 +0100505 for (b = 0; b < N; b++)
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100506 if(best[b].position == window-a)
Marc Kupietza77acce2017-11-30 16:59:07 +0100507 best[b].cprobability = best[b].activation / wpos_sum;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100508 } else {
509 printf("\x1b[1m%s\x1b[0m ", &vocab[d*max_w]);
510 }
Marc Kupietzbd41da32017-11-24 10:26:43 +0100511 pars->window_sums[a] = wpos_sum;
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100512 }
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100513 for (b = 0; b < pars->cutoff; b++)
Marc Kupietza77acce2017-11-30 16:59:07 +0100514 pars->target_sums[b] += target_sums[b]; //(target_sums[b] / wpos_sum ) / (window * 2);
515 printf("Target-Summe von 0: %f\n", pars->target_sums[150298]);
Marc Kupietzce3d4c62016-03-23 16:11:25 +0100516 free(target_sums);
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100517 for(b=0; b<N && best[b].wordi >= 0; b++);; // THIS LOOP IS NEEDED (b...)
518// printf("%d: best syn: %s %.2f %.5f\n", b, &vocab[best[b].wordi*max_w], best[b].activation, best[b].probability);
Marc Kupietzbd41da32017-11-24 10:26:43 +0100519// printf("\n");
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100520 nbs = malloc(sizeof(knn));
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100521 nbs->best = best;
Marc Kupietz271e2a42016-03-22 11:37:43 +0100522 nbs->length = b-1;
523 pthread_exit(nbs);
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100524}
525
Marc Kupietz30ca4342017-11-22 21:21:20 +0100526
Marc Kupietza2e64502016-04-27 09:53:51 +0200527wordlist *getTargetWords(char *st1, int search_backw) {
Marc Kupietz48c29682016-03-19 11:30:43 +0100528 wordlist *wl = malloc(sizeof(wordlist));
529 char st[100][max_size], sep[100];
530 long a, b=0, c=0, cn=0;
Marc Kupietza2e64502016-04-27 09:53:51 +0200531 int unmerged;
532
Marc Kupietzdc22b982015-10-09 09:19:34 +0200533 while (1) {
534 st[cn][b] = st1[c];
535 b++;
536 c++;
537 st[cn][b] = 0;
538 if (st1[c] == 0) break;
Marc Kupietz95aa1c02016-03-15 09:40:43 +0100539 if (st1[c] == ' ' || st1[c] == '-') {
540 sep[cn++] = st1[c];
Marc Kupietzdc22b982015-10-09 09:19:34 +0200541 b = 0;
542 c++;
543 }
544 }
545 cn++;
546 for (a = 0; a < cn; a++) {
Marc Kupietza2e64502016-04-27 09:53:51 +0200547 if(search_backw) {
548 for (b = words - 1; b >= 0; b--) if (!strcmp(&vocab[b * max_w], st[a])) break;
549 } else {
550 for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
551 }
Marc Kupietz34a3ee92016-02-27 22:43:16 +0100552 if (b == words) b = -1;
Marc Kupietz48c29682016-03-19 11:30:43 +0100553 wl->wordi[a] = b;
554 fprintf(stderr, "Word: \"%s\" Position in vocabulary: %lld\n", st[a], wl->wordi[a]);
Marc Kupietzdc22b982015-10-09 09:19:34 +0200555 if (b == -1) {
Marc Kupietze8da3062016-02-25 08:37:53 +0100556 fprintf(stderr, "Out of dictionary word!\n");
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100557 cn--;
Marc Kupietzf9ac54e2017-11-21 09:22:29 +0100558 free(wl);
559 return NULL;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200560 }
561 }
Marc Kupietz48c29682016-03-19 11:30:43 +0100562 wl->length=cn;
563 return(wl);
564}
565
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100566void *_get_neighbours(void *arg) {
567 knnpars *pars = arg;
Marc Kupietz48c29682016-03-19 11:30:43 +0100568 char *st1 = pars->token;
569 int N = pars->N;
570 long from = pars -> from;
571 unsigned long upto = pars -> upto;
572 char file_name[max_size], st[100][max_size], *sep;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100573 float dist, len, vec[max_size];
574 long long a, b, c, d, cn, *bi;
Marc Kupietz48c29682016-03-19 11:30:43 +0100575 char ch;
576 knn *nbs = NULL;
577 wordlist *wl = pars->wl;
578
Marc Kupietzc55c8872017-12-01 23:27:10 +0100579 collocator *best = pars->best;
Marc Kupietz48c29682016-03-19 11:30:43 +0100580
581 float worstbest=-1;
582
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100583 for (a = 0; a < N; a++) best[a].activation = 0;
Marc Kupietz48c29682016-03-19 11:30:43 +0100584 a = 0;
585 bi = wl->wordi;
586 cn = wl->length;
587 sep = wl->sep;
588 b = bi[0];
589 c = 0;
Marc Kupietz000ad862016-02-26 14:59:12 +0100590 if (b == -1) {
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100591 N = 0;
592 goto end;
593 }
Marc Kupietzdc22b982015-10-09 09:19:34 +0200594 for (a = 0; a < size; a++) vec[a] = 0;
595 for (b = 0; b < cn; b++) {
596 if (bi[b] == -1) continue;
Marc Kupietz95aa1c02016-03-15 09:40:43 +0100597 if(b>0 && sep[b-1] == '-')
598 for (a = 0; a < size; a++) vec[a] -= M[a + bi[b] * size];
599 else
600 for (a = 0; a < size; a++) vec[a] += M[a + bi[b] * size];
Marc Kupietzdc22b982015-10-09 09:19:34 +0200601 }
602 len = 0;
603 for (a = 0; a < size; a++) len += vec[a] * vec[a];
604 len = sqrt(len);
605 for (a = 0; a < size; a++) vec[a] /= len;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100606 for (a = 0; a < N; a++) best[a].activation = -1;
Marc Kupietz000ad862016-02-26 14:59:12 +0100607 for (c = from; c < upto; c++) {
Marc Kupietz5c3887d2016-04-28 08:53:35 +0200608 if(garbage && garbage[c]) continue;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200609 a = 0;
Marc Kupietz34020dc2016-02-25 08:44:19 +0100610// do not skip taget word
Marc Kupietze8da3062016-02-25 08:37:53 +0100611// for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
612// if (a == 1) continue;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200613 dist = 0;
614 for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
Marc Kupietzbe1b9fc2016-02-26 10:34:30 +0100615 if(dist > worstbest) {
616 for (a = 0; a < N; a++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100617 if (dist > best[a].activation) {
618 memmove(best + a + 1, best + a, (N - a -1) * sizeof(collocator));
619 best[a].activation = dist;
620 best[a].wordi = c;
Marc Kupietzbe1b9fc2016-02-26 10:34:30 +0100621 break;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200622 }
Marc Kupietzdc22b982015-10-09 09:19:34 +0200623 }
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100624 worstbest = best[N-1].activation;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200625 }
626 }
Marc Kupietz34020dc2016-02-25 08:44:19 +0100627
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100628end:
Marc Kupietz000ad862016-02-26 14:59:12 +0100629 pthread_exit(nbs);
Marc Kupietzdc22b982015-10-09 09:19:34 +0200630}
631
Marc Kupietze0e42132017-11-24 16:44:34 +0100632int cmp_activation (const void * a, const void * b) {
633 float fb = ((collocator *)a)->activation;
634 float fa = ((collocator *)b)->activation;
635 return (fa > fb) - (fa < fb);
636}
637
638int cmp_probability (const void * a, const void * b) {
639 float fb = ((collocator *)a)->probability;
640 float fa = ((collocator *)b)->probability;
641 return (fa > fb) - (fa < fb);
642}
643
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100644
Marc Kupietzd91212f2017-11-13 10:05:09 +0100645SV *get_neighbours(char *st1, int N, int sort_by, int search_backw, long cutoff, int dedupe) {
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100646 HV *result = newHV();
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100647 float *target_sums, vec[max_size];
Marc Kupietza5f60042017-05-04 10:38:12 +0200648 long long old_words;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100649 long a, b, c, d, slice;
Marc Kupietz271e2a42016-03-22 11:37:43 +0100650 knn *para_nbs[MAX_THREADS];
651 knn *syn_nbs[MAX_THREADS];
Marc Kupietz000ad862016-02-26 14:59:12 +0100652 knnpars pars[MAX_THREADS];
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100653 pthread_t *pt = (pthread_t *)malloc((num_threads+1) * sizeof(pthread_t));
Marc Kupietz48c29682016-03-19 11:30:43 +0100654 wordlist *wl;
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200655 int syn_threads = (M2? window * 2 : 0);
656 int para_threads = num_threads - syn_threads;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100657
Marc Kupietzc55c8872017-12-01 23:27:10 +0100658 collocator *best;
659 posix_memalign((void **) &best, 128, 10 * N * sizeof(collocator));
Marc Kupietzdb2dc7e2017-12-02 12:04:03 +0100660 memset(best, 0, 10 * N * sizeof(collocator));
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100661
Marc Kupietz000ad862016-02-26 14:59:12 +0100662 if(N>MAX_NEIGHBOURS) N=MAX_NEIGHBOURS;
663
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100664 if(cutoff < 1)
665 cutoff=words;
666
Marc Kupietza2e64502016-04-27 09:53:51 +0200667 wl = getTargetWords(st1, search_backw);
Marc Kupietzf9ac54e2017-11-21 09:22:29 +0100668 if(wl == NULL || wl->length < 1)
Marc Kupietz271e2a42016-03-22 11:37:43 +0100669 goto end;
Marc Kupietz48c29682016-03-19 11:30:43 +0100670
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100671 old_words = cutoff;
Marc Kupietza5f60042017-05-04 10:38:12 +0200672 if(merge_words > 0)
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100673 cutoff = merge_words * 1.25; /* HACK */
674 slice = cutoff / para_threads;
Marc Kupietza5f60042017-05-04 10:38:12 +0200675
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100676 a = posix_memalign((void **) &target_sums, 128, cutoff * sizeof(float));
677 for(a = 0; a < cutoff; a++)
Marc Kupietzce3d4c62016-03-23 16:11:25 +0100678 target_sums[a] = 0;
679
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200680 printf("Starting %d threads\n", para_threads);
681 fflush(stdout);
Marc Kupietz271e2a42016-03-22 11:37:43 +0100682 for(a=0; a < para_threads; a++) {
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100683 pars[a].cutoff = cutoff;
Marc Kupietz000ad862016-02-26 14:59:12 +0100684 pars[a].token = st1;
Marc Kupietz48c29682016-03-19 11:30:43 +0100685 pars[a].wl = wl;
Marc Kupietz000ad862016-02-26 14:59:12 +0100686 pars[a].N = N;
Marc Kupietzc55c8872017-12-01 23:27:10 +0100687 pars[a].best = &best[N*a];
Marc Kupietz000ad862016-02-26 14:59:12 +0100688 pars[a].from = a*slice;
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100689 pars[a].upto = ((a+1)*slice > cutoff? cutoff:(a+1)*slice);
Marc Kupietz000ad862016-02-26 14:59:12 +0100690 pthread_create(&pt[a], NULL, _get_neighbours, (void *) &pars[a]);
691 }
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200692 if(M2) {
693 for(a=0; a < syn_threads; a++) {
Marc Kupietz2c79c5e2017-11-09 16:18:40 +0100694 pars[a + para_threads].cutoff = cutoff;
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200695 pars[a + para_threads].target_sums = target_sums;
Marc Kupietzbd41da32017-11-24 10:26:43 +0100696 pars[a + para_threads].window_sums = window_sums;
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200697 pars[a + para_threads].wl = wl;
698 pars[a + para_threads].N = N;
699 pars[a + para_threads].from = a;
700 pars[a + para_threads].upto = a+1;
701 pthread_create(&pt[a + para_threads], NULL, getCollocators, (void *) &pars[a + para_threads]);
702 }
Marc Kupietz271e2a42016-03-22 11:37:43 +0100703 }
704 printf("Waiting for para threads to join\n");
705 fflush(stdout);
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100706 for (a = 0; a < para_threads; a++) pthread_join(pt[a], (void *) &para_nbs[a]);
Marc Kupietz271e2a42016-03-22 11:37:43 +0100707 printf("Para threads joint\n");
708 fflush(stdout);
Marc Kupietz000ad862016-02-26 14:59:12 +0100709
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200710 /* if(!syn_nbs[0]) */
711 /* goto end; */
Marc Kupietz000ad862016-02-26 14:59:12 +0100712
Marc Kupietzc55c8872017-12-01 23:27:10 +0100713 qsort(best, N*para_threads, sizeof(collocator), cmp_activation);
Marc Kupietz000ad862016-02-26 14:59:12 +0100714
Marc Kupietz000ad862016-02-26 14:59:12 +0100715
Marc Kupietzd0d3e912017-12-01 22:09:43 +0100716 long long chosen[MAX_NEIGHBOURS];
717 printf("N: %ld\n", N);
718
Marc Kupietz271e2a42016-03-22 11:37:43 +0100719 AV* array = newAV();
Marc Kupietzd91212f2017-11-13 10:05:09 +0100720 int i, j;
Marc Kupietza5f60042017-05-04 10:38:12 +0200721 int l1_words=0, l2_words=0;
Marc Kupietzd0d3e912017-12-01 22:09:43 +0100722
723 for (a = 0, i = 0; i < N && a < N*para_threads; a++) {
Marc Kupietzd91212f2017-11-13 10:05:09 +0100724 int filtered=0;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100725 long long c = best[a].wordi;
Marc Kupietzd91212f2017-11-13 10:05:09 +0100726 if (dedupe && i > 0) {
Marc Kupietzd0d3e912017-12-01 22:09:43 +0100727 for (j=0; j<i && !filtered; j++)
728 if (strcasestr(&vocab[c * max_w], &vocab[chosen[j] * max_w]) ||
729 strcasestr(&vocab[chosen[j] * max_w], &vocab[c * max_w])) {
730 printf("filtering %s %s\n", &vocab[chosen[j] * max_w], &vocab[c * max_w]);
Marc Kupietzd91212f2017-11-13 10:05:09 +0100731 filtered = 1;
732 }
733 if(filtered)
734 continue;
735 }
Marc Kupietza5f60042017-05-04 10:38:12 +0200736 if(merge_words > 0) {
737 if(c >= merge_words) {
738 if(l1_words > N / 2)
739 continue;
740 else
741 l1_words++;
742 } else {
743 if(l2_words > N / 2)
744 continue;
745 else
746 l2_words++;
747 }
748 }
Marc Kupietza5f60042017-05-04 10:38:12 +0200749 printf("%s l1:%d l2:%d i:%d a:%ld\n", &vocab[c * max_w], l1_words, l2_words, i, a);
Marc Kupietzd0d3e912017-12-01 22:09:43 +0100750 fflush(stdout);
Marc Kupietza5f60042017-05-04 10:38:12 +0200751
Marc Kupietz271e2a42016-03-22 11:37:43 +0100752 HV* hash = newHV();
Marc Kupietza5f60042017-05-04 10:38:12 +0200753 SV* word = newSVpvf(&vocab[c * max_w], 0);
Marc Kupietzd0d3e912017-12-01 22:09:43 +0100754 chosen[i] = c;
Marc Kupietz271e2a42016-03-22 11:37:43 +0100755 if(latin_enc == 0) SvUTF8_on(word);
Marc Kupietza5f60042017-05-04 10:38:12 +0200756 fflush(stdout);
Marc Kupietz271e2a42016-03-22 11:37:43 +0100757 hv_store(hash, "word", strlen("word"), word , 0);
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100758 hv_store(hash, "dist", strlen("dist"), newSVnv(best[a].activation), 0);
759 hv_store(hash, "rank", strlen("rank"), newSVuv(best[a].wordi), 0);
Marc Kupietz271e2a42016-03-22 11:37:43 +0100760 AV *vector = newAV();
761 for (b = 0; b < size; b++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100762 av_push(vector, newSVnv(M[b + best[a].wordi * size]));
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100763 }
Marc Kupietz271e2a42016-03-22 11:37:43 +0100764 hv_store(hash, "vector", strlen("vector"), newRV_noinc((SV*)vector), 0);
765 av_push(array, newRV_noinc((SV*)hash));
Marc Kupietza5f60042017-05-04 10:38:12 +0200766 i++;
Marc Kupietz271e2a42016-03-22 11:37:43 +0100767 }
768 hv_store(result, "paradigmatic", strlen("paradigmatic"), newRV_noinc((SV*)array), 0);
769
Marc Kupietz50485ba2016-03-23 09:13:14 +0100770 for(b=0; b < MAX_NEIGHBOURS; b++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100771 best[b].wordi = -1L;
772 best[b].activation = 0;
773 best[b].probability = 0;
774 best[b].position = 0;
775 best[b].activation_sum = 0;
Marc Kupietz50485ba2016-03-23 09:13:14 +0100776 }
777
Marc Kupietza77acce2017-11-30 16:59:07 +0100778 float total_activation = 0;
779
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200780 if (M2) {
781 printf("Waiting for syn threads to join\n");
782 fflush(stdout);
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100783 for (a = 0; a < syn_threads; a++) pthread_join(pt[a+para_threads], (void *) &syn_nbs[a]);
Marc Kupietza77acce2017-11-30 16:59:07 +0100784 for (a = 0; a <= syn_threads; a++) {
785 total_activation += window_sums[a];
786 printf("window pos: %d, sum: %f\n", a, window_sums[a]);
787 }
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200788 printf("syn threads joint\n");
789 fflush(stdout);
Marc Kupietz50485ba2016-03-23 09:13:14 +0100790
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200791 for(b=0; b < syn_nbs[0]->length; b++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100792 memcpy(best + b, &syn_nbs[0]->best[b], sizeof(collocator));
793 best[b].position = -1; // syn_nbs[0]->pos[b];
794 best[b].activation_sum = target_sums[syn_nbs[0]->best[b].wordi];
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100795 best[b].max_activation = 0.0;
Marc Kupietz4116b432017-12-06 14:15:32 +0100796 best[b].average = 0.0;
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100797 best[b].probability = 0.0;
Marc Kupietza77acce2017-11-30 16:59:07 +0100798 best[b].cprobability = syn_nbs[0]->best[b].cprobability;
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200799 }
Marc Kupietz30ca4342017-11-22 21:21:20 +0100800
801 float best_window_sum[MAX_NEIGHBOURS];
802 int found_index=0, i=0, j, w;
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100803 for(a=0; a < syn_threads; a++) {
804 for(b=0; b < syn_nbs[a]->length; b++) {
805 for(i=0; i < found_index; i++)
806 if(best[i].wordi == syn_nbs[a]->best[b].wordi)
807 break;
808 if(i >= found_index) {
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100809 best[found_index].max_activation = 0.0;
Marc Kupietz4116b432017-12-06 14:15:32 +0100810 best[found_index].average = 0.0;
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100811 best[found_index].probability = 0.0;
Marc Kupietza77acce2017-11-30 16:59:07 +0100812 best[found_index].cprobability = syn_nbs[a]->best[b].cprobability;
813 best[found_index].activation_sum = target_sums[syn_nbs[a]->best[b].wordi]; // syn_nbs[a]->best[b].activation_sum;
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100814 best[found_index++].wordi = syn_nbs[a]->best[b].wordi;
815 // printf("found: %s\n", &vocab[syn_nbs[a]->index[b] * max_w]);
Marc Kupietz30ca4342017-11-22 21:21:20 +0100816 }
817 }
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100818 }
Marc Kupietza77acce2017-11-30 16:59:07 +0100819 sort_by =0; // ALWAYS AUTO-FOCUS
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100820 if(sort_by != 1 && sort_by != 2) { // sort by auto focus mean
Marc Kupietz30ca4342017-11-22 21:21:20 +0100821 printf("window: %d - syn_threads: %d, %d\n", window, syn_threads, (1 << syn_threads) -1);
Marc Kupietzbd41da32017-11-24 10:26:43 +0100822 int wpos;
Marc Kupietz4116b432017-12-06 14:15:32 +0100823 int bits_set = 0;
Marc Kupietz30ca4342017-11-22 21:21:20 +0100824 for(i=0; i < found_index; i++) {
Marc Kupietz4116b432017-12-06 14:15:32 +0100825 best[i].activation = best[i].probability = best[i].average = best[i].cprobability_sum = 0;
Marc Kupietz30ca4342017-11-22 21:21:20 +0100826 for(w=1; w < (1 << syn_threads); w++) { // loop through all possible windows
Marc Kupietz4116b432017-12-06 14:15:32 +0100827 float word_window_sum = 0, word_window_average=0, word_cprobability_sum=0, word_activation_sum = 0, total_window_sum = 0;
828 bits_set = 0;
Marc Kupietzbd41da32017-11-24 10:26:43 +0100829 for(a=0; a < syn_threads; a++) {
Marc Kupietz30ca4342017-11-22 21:21:20 +0100830 if((1 << a) & w) {
Marc Kupietzbd41da32017-11-24 10:26:43 +0100831 wpos = (a >= window? a+1 : a);
832 total_window_sum += window_sums[wpos];
Marc Kupietz30ca4342017-11-22 21:21:20 +0100833 }
834 }
Marc Kupietzbd41da32017-11-24 10:26:43 +0100835// printf("%d window-sum %f\n", w, total_window_sum);
836 for(a=0; a < syn_threads; a++) {
837 if((1 << a) & w) {
838 wpos = (a >= window? a+1 : a);
839 bits_set++;
840 for(b=0; b < syn_nbs[a]->length; b++)
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100841 if(best[i].wordi == syn_nbs[a]->best[b].wordi) {
Marc Kupietza77acce2017-11-30 16:59:07 +0100842// float acti = syn_nbs[a]->best[b].activation / total_window_sum;
Marc Kupietzbd41da32017-11-24 10:26:43 +0100843// word_window_sum += syn_nbs[a]->dist[b] * syn_nbs[a]->norm[b]; // / window_sums[wpos]; // syn_nbs[a]->norm[b];
Marc Kupietzbd41da32017-11-24 10:26:43 +0100844// word_window_sum += syn_nbs[a]->norm[b]; // / window_sums[wpos]; // syn_nbs[a]->norm[b];
845// word_window_sum = (word_window_sum + syn_nbs[a]->norm[b]) - (word_window_sum * syn_nbs[a]->norm[b]); // syn_nbs[a]->norm[b];
Marc Kupietza77acce2017-11-30 16:59:07 +0100846
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100847 word_window_sum += syn_nbs[a]->best[b].activation; // / window_sums[wpos]; // syn_nbs[a]->norm[b];
Marc Kupietza77acce2017-11-30 16:59:07 +0100848// word_window_sum += acti - (word_window_sum * acti); syn_nbs[a]->best[b].activation; // / window_sums[wpos]; // syn_nbs[a]->norm[b];
849
Marc Kupietz4116b432017-12-06 14:15:32 +0100850 word_window_average += syn_nbs[a]->best[b].activation; // - word_window_average * syn_nbs[a]->best[b].activation; // conormalied activation sum
851 word_cprobability_sum += syn_nbs[a]->best[b].cprobability - word_cprobability_sum * syn_nbs[a]->best[b].cprobability; // conormalied column probability sum
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100852 word_activation_sum += syn_nbs[a]->best[b].activation;
853 if(syn_nbs[a]->best[b].activation > best[i].max_activation)
854 best[i].max_activation = syn_nbs[a]->best[b].activation;
Marc Kupietzdb2dc7e2017-12-02 12:04:03 +0100855 if(syn_nbs[a]->best[b].activation > best[i].heat[wpos] )
Marc Kupietz728b8ed2017-12-02 11:13:49 +0100856 best[i].heat[wpos] = syn_nbs[a]->best[b].activation;
Marc Kupietzbd41da32017-11-24 10:26:43 +0100857 }
858 }
859 }
Marc Kupietz4116b432017-12-06 14:15:32 +0100860 if(bits_set) {
861 word_window_average /= bits_set;
Marc Kupietzbd41da32017-11-24 10:26:43 +0100862// word_activation_sum /= bits_set;
863// word_window_sum /= bits_set;
Marc Kupietz4116b432017-12-06 14:15:32 +0100864 }
Marc Kupietza77acce2017-11-30 16:59:07 +0100865
866 word_window_sum /= total_window_sum;
Marc Kupietzbd41da32017-11-24 10:26:43 +0100867
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100868 if(word_window_sum > best[i].probability) {
Marc Kupietz4116b432017-12-06 14:15:32 +0100869// best[i].position = w;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100870 best[i].probability = word_window_sum;
Marc Kupietz30ca4342017-11-22 21:21:20 +0100871 }
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100872
Marc Kupietz4116b432017-12-06 14:15:32 +0100873 if(word_cprobability_sum > best[i].cprobability_sum) {
874 best[i].position = w;
875 best[i].cprobability_sum = word_cprobability_sum;
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100876 }
Marc Kupietz4116b432017-12-06 14:15:32 +0100877
878 best[i].average = word_window_average;
879// best[i].activation = word_activation_sum;
880 }
Marc Kupietz30ca4342017-11-22 21:21:20 +0100881 }
Marc Kupietze0e42132017-11-24 16:44:34 +0100882 qsort(best, found_index, sizeof(collocator), cmp_probability);
Marc Kupietz30ca4342017-11-22 21:21:20 +0100883// for(i=0; i < found_index; i++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100884// printf("found: %s - sum: %f - window: %d\n", &vocab[best[i].wordi * max_w], best[i].activation, best[i].position);
Marc Kupietz30ca4342017-11-22 21:21:20 +0100885// }
886
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100887 } else if(sort_by == 1) { // responsiveness any window position
888 int wpos;
889 for(i=0; i < found_index; i++) {
890 float word_window_sum = 0, word_activation_sum = 0, total_window_sum = 0;
891 for(a=0; a < syn_threads; a++) {
892 wpos = (a >= window? a+1 : a);
893 for(b=0; b < syn_nbs[a]->length; b++)
894 if(best[i].wordi == syn_nbs[a]->best[b].wordi) {
895 best[i].probability += syn_nbs[a]->best[b].probability;
896 if(syn_nbs[a]->best[b].activation > 0.25)
897 best[i].position |= 1 << wpos;
898 if(syn_nbs[a]->best[b].activation > best[i].activation) {
899 best[i].activation = syn_nbs[a]->best[b].activation;
900 }
901 }
902 }
903 }
904 qsort(best, found_index, sizeof(collocator), cmp_activation);
905 } else if(sort_by == 2) { // single window position
Marc Kupietz30ca4342017-11-22 21:21:20 +0100906 for(a=1; a < syn_threads; a++) {
907 for(b=0; b < syn_nbs[a]->length; b++) {
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200908 for(c=0; c < MAX_NEIGHBOURS; c++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100909 if(syn_nbs[a]->best[b].activation > best[c].activation) {
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200910 for(d=MAX_NEIGHBOURS-1; d>c; d--) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100911 memmove(best + d, best + d - 1, sizeof(collocator));
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200912 }
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100913 memcpy(best + c, &syn_nbs[a]->best[b], sizeof(collocator));
914 best[c].position = 1 << (-syn_nbs[a]->best[b].position+window - (syn_nbs[a]->best[b].position < 0 ? 1:0));
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200915 break;
Marc Kupietz6d9a6782016-03-23 17:25:25 +0100916 }
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200917 }
918 }
919 }
920 } else { // sort by mean p
921 for(a=1; a < syn_threads; a++) {
922 for(b=0; b < syn_nbs[a]->length; b++) {
923 for(c=0; c < MAX_NEIGHBOURS; c++) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100924 if(target_sums[syn_nbs[a]->best[b].wordi] > best[c].activation_sum) {
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200925 for(d=MAX_NEIGHBOURS-1; d>c; d--) {
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100926 memmove(best + d, best + d - 1, sizeof(collocator));
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200927 }
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100928 memcpy(best + c, &syn_nbs[a]->best[b], sizeof(collocator));
929 best[c].position = (1 << 2*window) - 1; // syn_nbs[a]->pos[b];
930 best[c].activation_sum = target_sums[syn_nbs[a]->best[b].wordi];
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200931 break;
932 }
Marc Kupietz271e2a42016-03-22 11:37:43 +0100933 }
Marc Kupietz6d9a6782016-03-23 17:25:25 +0100934 }
935 }
936 }
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200937 array = newAV();
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100938 for (a = 0, i=0; a < MAX_NEIGHBOURS && best[a].wordi >= 0; a++) {
939 long long c = best[a].wordi;
Marc Kupietza77acce2017-11-30 16:59:07 +0100940/*
Marc Kupietzd91212f2017-11-13 10:05:09 +0100941 if (dedupe) {
942 int filtered=0;
943 for (j=0; j<i; j++)
944 if (strcasestr(&vocab[c * max_w], chosen[j]) ||
945 strcasestr(chosen[j], &vocab[c * max_w])) {
Marc Kupietza77acce2017-11-30 16:59:07 +0100946 printf("filtering %s %s\n", chosen[j], &vocab[c * max_w]);
947 filtered = 1;
948 }
Marc Kupietzd91212f2017-11-13 10:05:09 +0100949 if(filtered)
950 continue;
951 }
Marc Kupietza77acce2017-11-30 16:59:07 +0100952*/
Marc Kupietzb6c615d2017-12-02 10:38:20 +0100953 chosen[i++]=c;
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200954 HV* hash = newHV();
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100955 SV* word = newSVpvf(&vocab[best[a].wordi * max_w], 0);
Marc Kupietzb6c615d2017-12-02 10:38:20 +0100956 AV* heat = newAV();
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200957 if(latin_enc == 0) SvUTF8_on(word);
958 hv_store(hash, "word", strlen("word"), word , 0);
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100959 hv_store(hash, "rank", strlen("rank"), newSVuv(best[a].wordi), 0);
Marc Kupietz4116b432017-12-06 14:15:32 +0100960 hv_store(hash, "average", strlen("average"), newSVnv(best[a].average), 0);
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100961 hv_store(hash, "prob", strlen("prob"), newSVnv(best[a].probability), 0);
Marc Kupietz4116b432017-12-06 14:15:32 +0100962 hv_store(hash, "cprob", strlen("cprob"), newSVnv(best[a].cprobability_sum), 0);
Marc Kupietzd64f3f22017-11-30 12:07:42 +0100963 hv_store(hash, "max", strlen("max"), newSVnv(best[a].max_activation), 0); // newSVnv(target_sums[best[a].wordi]), 0);
Marc Kupietza77acce2017-11-30 16:59:07 +0100964 hv_store(hash, "overall", strlen("overall"), newSVnv(best[a].activation_sum/total_activation), 0); // newSVnv(target_sums[best[a].wordi]), 0);
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100965 hv_store(hash, "pos", strlen("pos"), newSVnv(best[a].position), 0);
Marc Kupietzb6c615d2017-12-02 10:38:20 +0100966 best[a].heat[5]=0;
967 for(i=10; i >= 0; i--) av_push(heat, newSVnv(best[a].heat[i]));
968 hv_store(hash, "heat", strlen("heat"), newRV_noinc((SV*)heat), 0);
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200969 av_push(array, newRV_noinc((SV*)hash));
Marc Kupietz271e2a42016-03-22 11:37:43 +0100970 }
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200971 hv_store(result, "syntagmatic", strlen("syntagmatic"), newRV_noinc((SV*)array), 0);
Marc Kupietz271e2a42016-03-22 11:37:43 +0100972 }
Marc Kupietz000ad862016-02-26 14:59:12 +0100973end:
Marc Kupietz580ebdf2017-11-29 21:18:38 +0100974 words = old_words;
Marc Kupietzc8fdf3c2017-11-24 15:32:19 +0100975 free(best);
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100976 return newRV_noinc((SV*)result);
Marc Kupietz000ad862016-02-26 14:59:12 +0100977}
Marc Kupietz7bc85fd2016-02-24 11:42:41 +0100978
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200979int dump_vecs(char *fname) {
980 long i, j;
981 FILE *f;
982 /* if(words>200000) */
983 /* words=200000; */
Marc Kupietz6b2975c2016-03-18 21:59:33 +0100984
Marc Kupietz43ee87e2016-04-25 10:50:08 +0200985 if((f=fopen(fname, "w")) == NULL) {
986 fprintf(stderr, "cannot open %s for writing\n", fname);
987 return(-1);
988 }
989 fprintf(f, "%lld %lld\n", words, size);
990 for (i=0; i < words; i++) {
991 fprintf(f, "%s ", &vocab[i * max_w]);
992 for(j=0; j < size - 1; j++)
993 fprintf(f, "%f ", M[i*size + j]);
994 fprintf(f, "%f\n", M[i*size + j]);
995 }
996 fclose(f);
997 return(0);
998}
Marc Kupietzdc22b982015-10-09 09:19:34 +0200999