blob: c92400b76686cf6d2f60ed99e2b1dc7c01713f29 [file] [log] [blame]
Marc Kupietzdc22b982015-10-09 09:19:34 +02001#!/usr/local/bin/perl
2use Inline C;
3use Mojolicious::Lite;
Marc Kupietzc4893362016-02-25 08:04:46 +01004use Mojo::JSON qw(decode_json encode_json to_json);
Marc Kupietz247500f2015-10-09 11:29:01 +02005use Encode qw(decode encode);
Marc Kupietz7bc85fd2016-02-24 11:42:41 +01006use Mojo::Server::Daemon;
Marc Kupietzdc22b982015-10-09 09:19:34 +02007
Marc Kupietzc5990da2016-02-26 08:47:12 +01008print STDERR $ARGV[0];
Marc Kupietz7bc85fd2016-02-24 11:42:41 +01009# -cbow 1 -size 200 -window 8 -negative 25 -hs 0 -sample 1e-4 -threads 40 -binary 1 -iter 15
Marc Kupietz44bee3c2016-02-25 16:26:29 +010010init_net("vectors15.bin");
Marc Kupietzdc22b982015-10-09 09:19:34 +020011
12get '/' => sub {
13 my $c = shift;
14 my $word=$c->param('word');
Marc Kupietz44bee3c2016-02-25 16:26:29 +010015 my $no_nbs=$c->param('n') || 100;
16 my $no_iterations=$c->param('N') || 2000;
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +010017 my @lists;
Marc Kupietz7bc85fd2016-02-24 11:42:41 +010018 if(defined($word) && $word !~ /^\s*$/) {
19 $c->inactivity_timeout(300);
Marc Kupietz44bee3c2016-02-25 16:26:29 +010020 $word =~ s/\s+/ /g;
21 for my $w (split(' *\| *', $word)) {
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +010022 $c->app->log->debug('Looking for neighbours of '.$w);
23 push(@lists, get_neighbours(encode("iso-8859-1", $w), $no_nbs));
24 }
Marc Kupietz247500f2015-10-09 11:29:01 +020025 }
Marc Kupietz000ad862016-02-26 14:59:12 +010026 $word =~ s/ *\| */ | /g;
Marc Kupietz44bee3c2016-02-25 16:26:29 +010027 $c->render(template=>"index", word=>$word, no_nbs=>$no_nbs, no_iterations => $no_iterations, lists=> \@lists);
Marc Kupietzdc22b982015-10-09 09:19:34 +020028};
29
30app->start;
31
32exit;
33
34__END__
35
36__C__
37#include <stdio.h>
38#include <string.h>
39#include <math.h>
40#include <malloc.h>
41#include <stdlib.h> //strlen
Marc Kupietzf0809762016-02-26 10:13:47 +010042#include <sys/mman.h>
Marc Kupietz000ad862016-02-26 14:59:12 +010043#include <pthread.h>
Marc Kupietzdc22b982015-10-09 09:19:34 +020044
45#define max_size 2000
46#define max_w 50
Marc Kupietz7bc85fd2016-02-24 11:42:41 +010047#define MAX_NEIGHBOURS 1000
Marc Kupietz44bee3c2016-02-25 16:26:29 +010048#define MAX_WORDS -1
Marc Kupietz000ad862016-02-26 14:59:12 +010049#define MAX_THREADS 100
Marc Kupietzdc22b982015-10-09 09:19:34 +020050
51//the thread function
52void *connection_handler(void *);
Marc Kupietz000ad862016-02-26 14:59:12 +010053
54typedef struct {
55 long long *index;
56 float *dist;
57 unsigned int length;
58} knn;
59
60
61typedef struct {
62 char *token;
63 int N;
64 unsigned long from;
65 unsigned long upto;
66} knnpars;
67
Marc Kupietzdc22b982015-10-09 09:19:34 +020068float *M;
69char *vocab;
Marc Kupietz82b02672016-02-26 12:32:25 +010070long long words, size;
Marc Kupietz000ad862016-02-26 14:59:12 +010071int num_threads=20;
Marc Kupietzdc22b982015-10-09 09:19:34 +020072
73int init_net(char *file_name) {
Marc Kupietz67c20282016-02-26 09:42:00 +010074 FILE *f, *binvecs, *binwords;
Marc Kupietzf0809762016-02-26 10:13:47 +010075 int binwords_fd, binvecs_fd;
Marc Kupietz82b02672016-02-26 12:32:25 +010076 long long a, b, c, d, cn;
77 float len;
78
Marc Kupietz67c20282016-02-26 09:42:00 +010079 char binvecs_fname[256], binwords_fname[256];
80 strcpy(binwords_fname, file_name);
81 strcat(binwords_fname, ".words");
82 strcpy(binvecs_fname, file_name);
83 strcat(binvecs_fname, ".vecs");
Marc Kupietzdc22b982015-10-09 09:19:34 +020084
Marc Kupietzdc22b982015-10-09 09:19:34 +020085 f = fopen(file_name, "rb");
86 if (f == NULL) {
87 printf("Input file %s not found\n", file_name);
88 return -1;
89 }
90 fscanf(f, "%lld", &words);
Marc Kupietz44bee3c2016-02-25 16:26:29 +010091 if(MAX_WORDS > 0 && words > MAX_WORDS) words = MAX_WORDS;
Marc Kupietzdc22b982015-10-09 09:19:34 +020092 fscanf(f, "%lld", &size);
Marc Kupietzf0809762016-02-26 10:13:47 +010093 if( (binvecs_fd = open(binvecs_fname, O_RDONLY)) >= 0 && (binwords_fd = open(binwords_fname, O_RDONLY)) >= 0) {
Marc Kupietz000ad862016-02-26 14:59:12 +010094 M = mmap(0, sizeof(float) * (long long)words * (long long)size, PROT_READ, MAP_SHARED, binvecs_fd, 0);
95 vocab = mmap(0, sizeof(char) * (long long)words * max_w, PROT_READ, MAP_SHARED, binwords_fd, 0);
Marc Kupietzf0809762016-02-26 10:13:47 +010096 if (M == MAP_FAILED || vocab == MAP_FAILED) {
97 close(binvecs_fd);
98 close(binwords_fd);
99 fprintf(stderr, "Cannot mmap %s or %s\n", binwords_fname, binvecs_fname);
100 exit(-1);
101 }
Marc Kupietz67c20282016-02-26 09:42:00 +0100102 } else {
Marc Kupietzf0809762016-02-26 10:13:47 +0100103 vocab = (char *)malloc((long long)words * max_w * sizeof(char));
104 M = (float *)malloc((long long)words * (long long)size * sizeof(float));
105 if (M == NULL) {
106 printf("Cannot allocate memory: %lld MB %lld %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
107 return -1;
108 }
Marc Kupietz67c20282016-02-26 09:42:00 +0100109 for (b = 0; b < words; b++) {
110 a = 0;
111 while (1) {
112 vocab[b * max_w + a] = fgetc(f);
113 if (feof(f) || (vocab[b * max_w + a] == ' ')) break;
114 if ((a < max_w) && (vocab[b * max_w + a] != '\n')) a++;
115 }
116 vocab[b * max_w + a] = 0;
117 fread(&M[b * size], sizeof(float), size, f);
118 len = 0;
119 for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
120 len = sqrt(len);
121 for (a = 0; a < size; a++) M[a + b * size] /= len;
122 }
123 if( (binvecs = fopen(binvecs_fname, "wb")) != NULL && (binwords = fopen(binwords_fname, "wb")) != NULL) {
124 fwrite(M, sizeof(float), (long long)words * (long long)size, binvecs);
125 fclose(binvecs);
126 fwrite(vocab, sizeof(char), (long long)words * max_w, binwords);
127 fclose(binwords);
128 }
129 }
Marc Kupietzdc22b982015-10-09 09:19:34 +0200130 fclose(f);
131 return 0;
132}
133
Marc Kupietz000ad862016-02-26 14:59:12 +0100134
135void *_get_neighbours(knnpars *pars) {
136 char *st1 = pars->token;
137 int N = pars->N;
138 unsigned long from = pars -> from;
139 unsigned long upto = pars -> upto;
Marc Kupietz82b02672016-02-26 12:32:25 +0100140 char file_name[max_size], st[100][max_size];
Marc Kupietz000ad862016-02-26 14:59:12 +0100141 float dist, len, *bestd, vec[max_size];
142 long long a, b, c, d, cn, bi[100], *besti;
Marc Kupietz82b02672016-02-26 12:32:25 +0100143 char ch;
Marc Kupietz000ad862016-02-26 14:59:12 +0100144 knn *nbs = NULL;
Marc Kupietz82b02672016-02-26 12:32:25 +0100145
Marc Kupietz000ad862016-02-26 14:59:12 +0100146 besti = malloc(N * sizeof(long long));
147 bestd = malloc(N * sizeof(float));
Marc Kupietz7bc85fd2016-02-24 11:42:41 +0100148
Marc Kupietz000ad862016-02-26 14:59:12 +0100149 float worstbest=-1;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200150
151 for (a = 0; a < N; a++) bestd[a] = 0;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200152 a = 0;
153 cn = 0;
154 b = 0;
155 c = 0;
156 while (1) {
157 st[cn][b] = st1[c];
158 b++;
159 c++;
160 st[cn][b] = 0;
161 if (st1[c] == 0) break;
162 if (st1[c] == ' ') {
163 cn++;
164 b = 0;
165 c++;
166 }
167 }
168 cn++;
169 for (a = 0; a < cn; a++) {
170 for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
171 if (b == words) b = -1;
172 bi[a] = b;
Marc Kupietze8da3062016-02-25 08:37:53 +0100173 fprintf(stderr, "Word: \"%s\" Position in vocabulary: %lld\n", st[a], bi[a]);
Marc Kupietzdc22b982015-10-09 09:19:34 +0200174 if (b == -1) {
Marc Kupietze8da3062016-02-25 08:37:53 +0100175 fprintf(stderr, "Out of dictionary word!\n");
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100176 cn--;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200177 break;
178 }
179 }
Marc Kupietz000ad862016-02-26 14:59:12 +0100180 if (b == -1) {
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100181 N = 0;
182 goto end;
183 }
Marc Kupietzdc22b982015-10-09 09:19:34 +0200184 for (a = 0; a < size; a++) vec[a] = 0;
185 for (b = 0; b < cn; b++) {
186 if (bi[b] == -1) continue;
187 for (a = 0; a < size; a++) vec[a] += M[a + bi[b] * size];
188 }
189 len = 0;
190 for (a = 0; a < size; a++) len += vec[a] * vec[a];
191 len = sqrt(len);
192 for (a = 0; a < size; a++) vec[a] /= len;
193 for (a = 0; a < N; a++) bestd[a] = -1;
Marc Kupietz000ad862016-02-26 14:59:12 +0100194 for (c = from; c < upto; c++) {
Marc Kupietzdc22b982015-10-09 09:19:34 +0200195 a = 0;
Marc Kupietz34020dc2016-02-25 08:44:19 +0100196// do not skip taget word
Marc Kupietze8da3062016-02-25 08:37:53 +0100197// for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
198// if (a == 1) continue;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200199 dist = 0;
200 for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
Marc Kupietzbe1b9fc2016-02-26 10:34:30 +0100201 if(dist > worstbest) {
202 for (a = 0; a < N; a++) {
203 if (dist > bestd[a]) {
204 for (d = N - 1; d > a; d--) {
205 bestd[d] = bestd[d - 1];
206 besti[d] = besti[d - 1];
207 }
208 bestd[a] = dist;
209 besti[a] = c;
210 break;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200211 }
Marc Kupietzdc22b982015-10-09 09:19:34 +0200212 }
Marc Kupietzbe1b9fc2016-02-26 10:34:30 +0100213 worstbest = bestd[N-1];
Marc Kupietzdc22b982015-10-09 09:19:34 +0200214 }
215 }
Marc Kupietz34020dc2016-02-25 08:44:19 +0100216
Marc Kupietz000ad862016-02-26 14:59:12 +0100217 nbs = malloc(sizeof(knn));
218 nbs->index = besti;
219 nbs->dist = bestd;
220 nbs->length = N;
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100221end:
Marc Kupietz000ad862016-02-26 14:59:12 +0100222 pthread_exit(nbs);
Marc Kupietzdc22b982015-10-09 09:19:34 +0200223}
224
Marc Kupietz000ad862016-02-26 14:59:12 +0100225SV *get_neighbours(char *st1, int N) {
226 float bestd[MAX_NEIGHBOURS], vec[max_size];
227 long long besti[MAX_NEIGHBOURS], a, b, c, d, slice;
228 char *bestw[MAX_NEIGHBOURS];
229 knn *nbs[MAX_THREADS];
230 knnpars pars[MAX_THREADS];
231 pthread_t *pt = (pthread_t *)malloc(num_threads * sizeof(pthread_t));
232 AV* array = newAV();
233
234 if(N>MAX_NEIGHBOURS) N=MAX_NEIGHBOURS;
235
236 slice = words / num_threads;
237
238 for(a=0; a < num_threads; a++) {
239 pars[a].token = st1;
240 pars[a].N = N;
241 pars[a].from = a*slice;
242 pars[a].upto = ((a+1)*slice > words? words:(a+1)*slice);
243 pthread_create(&pt[a], NULL, _get_neighbours, (void *) &pars[a]);
244 }
245 for (a = 0; a < num_threads; a++) pthread_join(pt[a], &nbs[a]);
246
247 if(!nbs[0])
248 goto end;
249
250 for(b=0; b < N; b++) {
251 besti[b] = nbs[0]->index[b];
252 bestd[b] = nbs[0]->dist[b];
253 }
254
255 for(a=1; a < num_threads; a++) {
256 for(b=0; b < N; b++) {
257 for(c=0; c < N; c++) {
258 if(nbs[a]->dist[b] > bestd[c]) {
259 for(d=N-1; d>c; d--) {
260 bestd[d] = bestd[d-1];
261 besti[d] = besti[d-1];
262 }
263 besti[c] = nbs[a]->index[b];
264 bestd[c] = nbs[a]->dist[b];
265 break;
266 }
267 }
268 }
269 }
270
271 if(nbs) {
272 for (a = 0; a < N; a++) {
273 bestw[a] = (char *)malloc(max_size * sizeof(char));
274 }
275 for (a = 0; a < N; a++) {
276 strcpy(bestw[a], &vocab[besti[a] * max_w]);
277 HV* hash = newHV();
278 hv_store(hash, "word", strlen("word"), newSVpvf(bestw[a], 0), 0);
279 hv_store(hash, "dist", strlen("dist"), newSVnv(bestd[a]), 0);
280 hv_store(hash, "rank", strlen("rank"), newSVuv(besti[a]), 0);
281 AV *vector = newAV();
282 for (b = 0; b < size; b++) {
283 av_push(vector, newSVnv(M[b + besti[a] * size]));
284 }
285 hv_store(hash, "vector", strlen("vector"), newRV_noinc((SV*)vector), 0);
286 av_push(array, newRV_noinc((SV*)hash));
287 }
288 }
289end:
290 return newRV_noinc((SV*)array);
291}
Marc Kupietz7bc85fd2016-02-24 11:42:41 +0100292
Marc Kupietzdc22b982015-10-09 09:19:34 +0200293__DATA__
294
295@@ index.html.ep
296<!DOCTYPE html>
297<html>
Marc Kupietzc4893362016-02-25 08:04:46 +0100298<head>
299 <title>DeReKo-Word-Vector-Distances</title>
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100300 <link rel="stylesheet" href="//code.jquery.com/ui/1.11.4/themes/smoothness/jquery-ui.css">
Marc Kupietzc4893362016-02-25 08:04:46 +0100301 <script src="http://code.jquery.com/jquery-latest.min.js"></script>
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100302 <script src="//code.jquery.com/ui/1.11.4/jquery-ui.js"></script>
303 <script>
304 $(function() {
Marc Kupietz5f780672016-02-25 17:15:54 +0100305 $( document ).tooltip({
306 content: function() {
307 return $(this).attr('title');
308 }}
309 )
310 })
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100311 </script>
Marc Kupietzc4893362016-02-25 08:04:46 +0100312 <script src="//d3js.org/d3.v3.min.js" charset="utf-8"></script>
313 <script src="http://klinux10/word2vec/tsne.js"></script>
Marc Kupietzc5990da2016-02-26 08:47:12 +0100314 <script src="http://klinux10/word2vec/labeler.js"></script>
Marc Kupietzc4893362016-02-25 08:04:46 +0100315<style>
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100316body, input {
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100317 font-family: Arial, sans-serif;
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100318 font-size: 11pt;
319}
320
321.ui-tooltip-content {
322 font-size: 9pt;
323 colour: #222222;
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100324}
Marc Kupietz5f780672016-02-25 17:15:54 +0100325
326svg > .ui-tooltip-content {
Marc Kupietzb1029362016-02-27 21:38:55 +0100327 font-size: 8pt;
Marc Kupietz5f780672016-02-25 17:15:54 +0100328 colour: #222222;
329}
330
Marc Kupietzc4893362016-02-25 08:04:46 +0100331svg {
Marc Kupietzb1029362016-02-27 21:38:55 +0100332 margin-right: 10px;
333 margin-bottom: 10px;
Marc Kupietzc4893362016-02-25 08:04:46 +0100334}
335#wrapper {
336 width: 100%;
337// border: 1px solid red;
338 overflow: hidden; /* will contain if #first is longer than #second */
339}
340#first {
Marc Kupietzb1029362016-02-27 21:38:55 +0100341 margin-right: 20px;
342 float: left;
343 // border: 1px solid green;
Marc Kupietzc4893362016-02-25 08:04:46 +0100344}
345#second {
346 border: 1px solid #333;
347 overflow: hidden; /* if you don't want #second to wrap below #first */
348}
Marc Kupietz4aa62172016-02-25 10:39:27 +0100349#cost {
Marc Kupietzb1029362016-02-27 21:38:55 +0100350 font-size: 8pt;
351 color: #222222;
352 margin-top: 4px;
Marc Kupietz4aa62172016-02-25 10:39:27 +0100353}
Marc Kupietzc4893362016-02-25 08:04:46 +0100354</style>
355<script>
356
Marc Kupietz4aa62172016-02-25 10:39:27 +0100357var opt = {epsilon: 1, perplexity: 20};
Marc Kupietzc4893362016-02-25 08:04:46 +0100358var T = new tsnejs.tSNE(opt); // create a tSNE instance
359
360var Y;
361
362var data;
Marc Kupietzc5990da2016-02-26 08:47:12 +0100363var labeler;
Marc Kupietzc4893362016-02-25 08:04:46 +0100364
Marc Kupietzc5990da2016-02-26 08:47:12 +0100365
366function applyJitter() {
367 svg.selectAll('.u')
368 .data(labels)
369 .transition()
370 .duration(50)
371 .attr("transform", function(d, i) {
372 T.Y[i][0] = (d.x - 400 - tx)/ss/20;
373 T.Y[i][1] = (d.y - 400 - ty)/ss/20;
374 return "translate(" +
375 (d.x) + "," +
376 (d.y) + ")";
377 });
378}
379
Marc Kupietzc4893362016-02-25 08:04:46 +0100380function updateEmbedding() {
381 var Y = T.getSolution();
382 svg.selectAll('.u')
383 .data(data.words)
Marc Kupietzc5990da2016-02-26 08:47:12 +0100384 .attr("transform", function(d, i) {
385 return "translate(" +
386 ((Y[i][0]*20*ss + tx) + 400) + "," +
387 ((Y[i][1]*20*ss + ty) + 400) + ")"; });
Marc Kupietzc4893362016-02-25 08:04:46 +0100388}
389
390var svg;
Marc Kupietzc5990da2016-02-26 08:47:12 +0100391var labels = [];
392var anchor_array = [];
393var text;
394
Marc Kupietzc4893362016-02-25 08:04:46 +0100395function drawEmbedding() {
Marc Kupietza350bce2016-02-25 09:34:25 +0100396 $("#embed").empty();
397 var div = d3.select("#embed");
398
399 // get min and max in each column of Y
400 var Y = T.Y;
401
402 svg = div.append("svg") // svg is global
403 .attr("width", 800)
404 .attr("height", 800);
405
406 var g = svg.selectAll(".b")
407 .data(data.words)
408 .enter().append("g")
409 .attr("class", "u");
410
411 g.append("a")
412 .attr("xlink:href", function(word) {return "/?word="+word;})
Marc Kupietz5f780672016-02-25 17:15:54 +0100413 .attr("title", function(d, i) {
414 return "rank: "+i +" "+"freq. rank: "+data.ranks[i];
415 })
Marc Kupietza350bce2016-02-25 09:34:25 +0100416 .append("text")
417 .attr("text-anchor", "top")
418 .attr("font-size", 12)
419 .attr("fill", function(d) {
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100420 if(data.target.indexOf(" "+d+" ") >= 0) {
Marc Kupietza350bce2016-02-25 09:34:25 +0100421 return "red";
422 } else {
423 return "#333"
Marc Kupietzc4893362016-02-25 08:04:46 +0100424 }
Marc Kupietza350bce2016-02-25 09:34:25 +0100425 })
426 .text(function(d) { return d; });
427
428 var zoomListener = d3.behavior.zoom()
429 .scaleExtent([0.1, 10])
430 .center([0,0])
431 .on("zoom", zoomHandler);
432 zoomListener(svg);
433 }
434
435 var tx=0, ty=0;
436 var ss=1;
437 var iter_id=-1;
438
439 function zoomHandler() {
440 tx = d3.event.translate[0];
441 ty = d3.event.translate[1];
442 ss = d3.event.scale;
443 updateEmbedding();
444 }
445
446 var stepnum = 0;
Marc Kupietzc5990da2016-02-26 08:47:12 +0100447
Marc Kupietza350bce2016-02-25 09:34:25 +0100448 function stopStep() {
449 clearInterval(iter_id);
Marc Kupietzc5990da2016-02-26 08:47:12 +0100450 text = svg.selectAll("text");
451
452 labels = d3.range(data.words.length).map(function(i) {
453 var x = (T.Y[i][0]*20*ss + tx) + 400;
454 var y = (T.Y[i][1]*20*ss + ty) + 400;
455 anchor_array.push({x: x, y: y, r: 5});
456 return {
457 x: x,
458 y: y,
459 name: data.words[i]
460 };
461 });
462
463 var index = 0;
464 text.each(function() {
465 labels[index].width = this.getBBox().width;
466 labels[index].height = this.getBBox().height;
467 index += 1;
468 });
469
470
471// setTimeout(updateEmbedding, 1);
472// setTimeout(
473 labeler = d3.labeler()
474 .label(labels)
475 .anchor(anchor_array)
476 .width(800)
477 .height(800)
478 .update(applyJitter);
479// .start(1000);
480
481 iter_id = setInterval(jitterStep, 1);
482
Marc Kupietza350bce2016-02-25 09:34:25 +0100483 }
Marc Kupietzc5990da2016-02-26 08:47:12 +0100484
485var jitter_i=0;;
486
487function jitterStep() {
488 if(jitter_i++ > 100) {
489 clearInterval(iter_id);
490 } else {
491 labeler.start2(10);
492 applyJitter();
493 }
494}
Marc Kupietzb1029362016-02-27 21:38:55 +0100495
496var last_cost=1000;
497
Marc Kupietza350bce2016-02-25 09:34:25 +0100498 function step() {
499 var i = T.iter;
Marc Kupietzb1029362016-02-27 21:38:55 +0100500
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100501 if(i > <%= $no_iterations %>) {
Marc Kupietza350bce2016-02-25 09:34:25 +0100502 stopStep();
503 } else {
Marc Kupietzb1029362016-02-27 21:38:55 +0100504 var cost = Math.round(T.step() *100000) / 100000; // do a few steps
505 $("#cost").html("tsne iteration " + i + ", cost: " + cost.toFixed(5));
506 if(i % 250 == 0 && cost >= last_cost) {
507 stopStep();
508 } else {
509 last_cost = cost;
510 updateEmbedding();
511 }
Marc Kupietza350bce2016-02-25 09:34:25 +0100512 }
513 }
Marc Kupietzc5990da2016-02-26 08:47:12 +0100514
Marc Kupietza350bce2016-02-25 09:34:25 +0100515 function showMap(j) {
516 data=j;
517 T.iter=0;
518 T.initDataRaw(data.vecs); // init embedding
519 drawEmbedding(); // draw initial embedding
520
521 if(iter_id >= 0) {
522 clearInterval(iter_id);
523 }
524 //T.debugGrad();
525 iter_id = setInterval(step, 1);
526 //step();
527 }
528
Marc Kupietzc5990da2016-02-26 08:47:12 +0100529 function update() {
530
531 text.transition().duration(800)
532 .attr("transform", transform);
533 }
534
535
Marc Kupietzc4893362016-02-25 08:04:46 +0100536</script>
537</head>
Marc Kupietzdc22b982015-10-09 09:19:34 +0200538<body>
Marc Kupietz4aa62172016-02-25 10:39:27 +0100539 <form action="<%=url_for('/')->to_abs%>" method="GET">
Marc Kupietz44bee3c2016-02-25 16:26:29 +0100540 word(s):
541 <input type="text" name="word" size="20" value="<%= $word %>" title="When looking for multiple words use spaces as separators to search around the average vector and | as separator to get the neighbours for each word.">
542 max. neighbours: <input type="text" size="8" name="n" value="<%= $no_nbs %>">
543 iterations: <input type="text" name="N" size="8" value="<%= $no_iterations %>">
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100544 <input type="submit" value="Show">
Marc Kupietz4aa62172016-02-25 10:39:27 +0100545 </form>
546 <br>
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100547 % if($lists) {
Marc Kupietz4aa62172016-02-25 10:39:27 +0100548 <div id="wrapper">
549 <table id="first">
550 <tr>
Marc Kupietz5f780672016-02-25 17:15:54 +0100551 <th align="right">Pos.</th><th align="left">Word</th><th align="right">Cosine dist.</th><th>Freq. rank</th>
Marc Kupietz4aa62172016-02-25 10:39:27 +0100552 </tr>
Marc Kupietz5f780672016-02-25 17:15:54 +0100553 % my $j=0; my @words; my @vecs; my @ranks; for my $list (@$lists) {
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100554 % my $i=1; for my $item (@$list) {
555 % if(!grep{$_ eq $item->{word}} @words) {
556 % push @vecs, $item->{vector};
557 % push @words, $item->{word};
Marc Kupietz5f780672016-02-25 17:15:54 +0100558 % push @ranks, $item->{rank};
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100559 % }
Marc Kupietz4aa62172016-02-25 10:39:27 +0100560 <tr>
561 <td align="right">
562 <%= $i++ %>.
563 </td>
564 <td>
565 <a href="/?word=<%= $item->{word} %>">
566 <%= $item->{word} %>
567 </a>
568 </td>
569 <td align="right">
570 <%= sprintf("%.3f", $item->{dist}) %>
571 </td>
Marc Kupietz5f780672016-02-25 17:15:54 +0100572 <td align="right">
573 <%= $item->{rank} %>
574 </td>
Marc Kupietz4aa62172016-02-25 10:39:27 +0100575 </tr>
576 % }
Marc Kupietz7b2cbeb2016-02-25 11:22:00 +0100577 % }
Marc Kupietz4aa62172016-02-25 10:39:27 +0100578 </table>
579 <script>
580 % use Mojo::ByteStream 'b';
581 $(window).load(function() {
Marc Kupietz5f780672016-02-25 17:15:54 +0100582 showMap(<%= b(Mojo::JSON::to_json({target => " $word ", words => \@words, vecs => \@vecs, ranks => \@ranks})); %>);
Marc Kupietz4aa62172016-02-25 10:39:27 +0100583 });
584 </script>
585 % }
586 <div id="second" style="width:800px; height:800px; font-family: arial;">
587 <div id="embed">
588 </div>
Marc Kupietz4aa62172016-02-25 10:39:27 +0100589 </div>
Marc Kupietzb1029362016-02-27 21:38:55 +0100590 <div id="cost"></div>
Marc Kupietz4aa62172016-02-25 10:39:27 +0100591 </div>
592 <p>
593 Word vector model based on DeReKo-2015-II. Trained with <a href="https://code.google.com/p/word2vec/">word2vec</a> using the following parameters:</p>
Marc Kupietz247500f2015-10-09 11:29:01 +0200594 <pre>
Marc Kupietz7bc85fd2016-02-24 11:42:41 +0100595-cbow 1 -size 300 -window 7 -negative 5 -hs 0 -sample 1e-5 -threads 44 -binary 1 -iter 5
Marc Kupietz4aa62172016-02-25 10:39:27 +0100596 </pre>
597 </p>
598</body>
Marc Kupietzdc22b982015-10-09 09:19:34 +0200599</html>
600