Gitiles
Code Review Sign In
korap.ids-mannheim.de / ids-kl / derekovecs / 247500f4ca00e4ec3fd71efebf8d0a3d87ebd102 / . / w2v-server.pl
blob: 82e7b17cf2b32b8ecf687c4c08d8bdb53cc05afd [file] [log] [blame]
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]1#!/usr/local/bin/perl
2use Inline C;
3use Mojolicious::Lite;
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]4use Encode qw(decode encode);
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]5
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]6init_net("vectors14.bin");
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]7
8get '/' => sub {
9 my $c = shift;
10 my $word=$c->param('word');
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]11 my $list;
12 if($word !~ /^\s*$/) {
13 $list = print_neighbours(encode("iso-8859-1", $word));
14 }
15 $c->render(template=>"index", word=>$word, list=> $list);
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]16};
17
18app->start;
19
20exit;
21
22__END__
23
24__C__
25#include <stdio.h>
26#include <string.h>
27#include <math.h>
28#include <malloc.h>
29#include <stdlib.h> //strlen
30
31#define max_size 2000
32#define max_w 50
33#define N 75
34
35//the thread function
36void *connection_handler(void *);
37
38char *bestw[N];
39char file_name[max_size], st[100][max_size];
40float dist, len, bestd[N], vec[max_size];
41long long words, size, a, b, c, d, cn, bi[100];
42char ch;
43float *M;
44char *vocab;
45char *stringBuffer;
46
47int init_net(char *file_name) {
48 FILE *f;
49
50 stringBuffer = malloc(64000);
51 f = fopen(file_name, "rb");
52 if (f == NULL) {
53 printf("Input file %s not found\n", file_name);
54 return -1;
55 }
56 fscanf(f, "%lld", &words);
57 fscanf(f, "%lld", &size);
58 vocab = (char *)malloc((long long)words * max_w * sizeof(char));
59 for (a = 0; a < N; a++) bestw[a] = (char *)malloc(max_size * sizeof(char));
60 M = (float *)malloc((long long)words * (long long)size * sizeof(float));
61 if (M == NULL) {
62 printf("Cannot allocate memory: %lld MB %lld %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
63 return -1;
64 }
65 for (b = 0; b < words; b++) {
66 a = 0;
67 while (1) {
68 vocab[b * max_w + a] = fgetc(f);
69 if (feof(f) || (vocab[b * max_w + a] == ' ')) break;
70 if ((a < max_w) && (vocab[b * max_w + a] != '\n')) a++;
71 }
72 vocab[b * max_w + a] = 0;
73 for (a = 0; a < size; a++) fread(&M[a + b * size], sizeof(float), 1, f);
74 len = 0;
75 for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
76 len = sqrt(len);
77 for (a = 0; a < size; a++) M[a + b * size] /= len;
78 }
79 fclose(f);
80 return 0;
81}
82
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]83SV *print_neighbours(char *st1) {
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]84 FILE *out=stdout;
85 *stringBuffer=0;
86
87 for (a = 0; a < N; a++) bestd[a] = 0;
88 for (a = 0; a < N; a++) bestw[a][0] = 0;
89 a = 0;
90 cn = 0;
91 b = 0;
92 c = 0;
93 while (1) {
94 st[cn][b] = st1[c];
95 b++;
96 c++;
97 st[cn][b] = 0;
98 if (st1[c] == 0) break;
99 if (st1[c] == ' ') {
100 cn++;
101 b = 0;
102 c++;
103 }
104 }
105 cn++;
106 for (a = 0; a < cn; a++) {
107 for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
108 if (b == words) b = -1;
109 bi[a] = b;
110 sprintf(stringBuffer, "\n<pre>Word: \"%s\" Position in vocabulary: %lld</pre>\n", st[a], bi[a]);
111 if (b == -1) {
112 sprintf(stringBuffer+strlen(stringBuffer), "Out of dictionary word!\n");
113 break;
114 }
115 }
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]116 if (b == -1) goto end;
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]117 for (a = 0; a < size; a++) vec[a] = 0;
118 for (b = 0; b < cn; b++) {
119 if (bi[b] == -1) continue;
120 for (a = 0; a < size; a++) vec[a] += M[a + bi[b] * size];
121 }
122 len = 0;
123 for (a = 0; a < size; a++) len += vec[a] * vec[a];
124 len = sqrt(len);
125 for (a = 0; a < size; a++) vec[a] /= len;
126 for (a = 0; a < N; a++) bestd[a] = -1;
127 for (a = 0; a < N; a++) bestw[a][0] = 0;
128 for (c = 0; c < words; c++) {
129 a = 0;
130 for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
131 if (a == 1) continue;
132 dist = 0;
133 for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
134 for (a = 0; a < N; a++) {
135 if (dist > bestd[a]) {
136 for (d = N - 1; d > a; d--) {
137 bestd[d] = bestd[d - 1];
138 strcpy(bestw[d], bestw[d - 1]);
139 }
140 bestd[a] = dist;
141 strcpy(bestw[a], &vocab[c * max_w]);
142 break;
143 }
144 }
145 }
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]146 AV* array = newAV();
147 for (a = 0; a < N; a++) {
148 HV* hash = newHV();
149 hv_store(hash, "word", strlen("word"), newSVpvf(bestw[a], 0), 0);
150 hv_store(hash, "dist", strlen("dist"), newSVnv(bestd[a]), 0);
151 av_push(array, newRV_noinc((SV*)hash));
152 }
153 end:
154 return newRV_noinc((SV*)array);
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]155}
156
157__DATA__
158
159@@ index.html.ep
160<!DOCTYPE html>
161<html>
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]162<head><title>DeReKo-Word-Vector-Distances</title></head>
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]163<body>
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]164 <p>Word vector model based on a 1.9 billion word sample of DeReKo-2015-II (mainly wikipedia including discussions, current newspapapers and fiction). Trained with <a href="https://code.google.com/p/word2vec/">word2vec</a> using the following parameters:</p>
165 <pre>
166-cbow 1 -size 200 -window 8 -negative 25 -hs 0 -sample 1e-4 -threads 40 -binary 1 -iter 15
167</pre>
168</p>
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]169 <form action="<%=url_for('/')->to_abs%>" method="GET">
170 Word: <input type="text" name="word">
171 <input type="submit" value="Show neighbours">
172 </form>
173 <br>
Marc Kupietz247500f2015-10-09 11:29:01 +0200[diff] [blame^]174 % if($list) {
175 <p>Target word: <b><%= $word%></b></p>
176 <h3>Nearest neighbours</h3>
177 <table>
178 <tr>
179 <th align="right">Pos.</th><th align="left">Word</th><th align="right">Cosine dist.</th>
180 </tr>
181 % my $i=1; for my $item (@$list) {
182 <tr>
183 <td align="right">
184 <%= $i++ %>.
185 </td>
186 <td>
187 <a href="/?word=<%= $item->{word} %>">
188 <%= $item->{word} %>
189 </a>
190 </td>
191 <td align="right">
192 <%= sprintf("%.3f", $item->{dist}) %>
193 </td>
194 </tr>
195 % }
196 </table>
197 % }
Marc Kupietzdc22b982015-10-09 09:19:34 +0200[diff] [blame]198</body>
199</html>
200