w2v-server.pl - ids-kl/derekovecs - Gitiles

 #!/usr/local/bin/perl
 use Inline C;
 use Mojolicious::Lite;
 use Mojo::JSON qw(decode_json encode_json to_json);
 use Encode qw(decode encode);
 use Mojo::Server::Daemon;

 # -cbow 1 -size 200 -window 8 -negative 25 -hs 0 -sample 1e-4 -threads 40 -binary 1 -iter 15
 init_net("vectors14.bin");

 get '/' => sub {
   my $c    = shift;
 	my $word=$c->param('word');
 	my $no_nbs=$c->param('n') || 50;
 	my $list;
 	if(defined($word) && $word !~ /^\s*$/) {
 		$c->inactivity_timeout(300);
 		$c->app->log->debug('Looking for neighbours of '.$word);
 		$list = get_neighbours(encode("iso-8859-1", $word), $no_nbs);
 	}
   $c->render(template=>"index", word=>$word, no_nbs=>$no_nbs, list=> $list);
 };

 app->start;

 exit;

 __END__

 __C__
 #include <stdio.h>
 #include <string.h>
 #include <math.h>
 #include <malloc.h>
 #include <stdlib.h>    //strlen

 #define max_size 2000
 #define max_w 50
 #define MAX_NEIGHBOURS 1000

 //the thread function
 void *connection_handler(void *);

 char *bestw[MAX_NEIGHBOURS];
 char file_name[max_size], st[100][max_size];
 float dist, len, bestd[MAX_NEIGHBOURS], vec[max_size];
 long long words, size, a, b, c, d, cn, bi[100], besti[MAX_NEIGHBOURS];
 char ch;
 float *M;
 char *vocab;
 char *stringBuffer;

 int init_net(char *file_name) {
   FILE *f;

 	stringBuffer = malloc(64000);
   f = fopen(file_name, "rb");
   if (f == NULL) {
     printf("Input file %s not found\n", file_name);
     return -1;
   }
   fscanf(f, "%lld", &words);
   fscanf(f, "%lld", &size);
   vocab = (char *)malloc((long long)words * max_w * sizeof(char));
   for (a = 0; a < MAX_NEIGHBOURS; a++) bestw[a] = (char *)malloc(max_size * sizeof(char));
   M = (float *)malloc((long long)words * (long long)size * sizeof(float));
   if (M == NULL) {
     printf("Cannot allocate memory: %lld MB    %lld  %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
     return -1;
   }
   for (b = 0; b < words; b++) {
     a = 0;
     while (1) {
       vocab[b * max_w + a] = fgetc(f);
       if (feof(f) || (vocab[b * max_w + a] == ' ')) break;
       if ((a < max_w) && (vocab[b * max_w + a] != '\n')) a++;
     }
     vocab[b * max_w + a] = 0;
     fread(&M[b * size], sizeof(float), size, f);
     len = 0;
     for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
     len = sqrt(len);
     for (a = 0; a < size; a++) M[a + b * size] /= len;
   }
   fclose(f);
 	return 0;
 }

 SV *get_neighbours(char *st1, int N) {
   if(N>MAX_NEIGHBOURS) N=MAX_NEIGHBOURS;

 	FILE *out=stdout;
 	*stringBuffer=0;

 	for (a = 0; a < N; a++) bestd[a] = 0;
 	for (a = 0; a < N; a++) bestw[a][0] = 0;
 	a = 0;
 	cn = 0;
 	b = 0;
 	c = 0;
 	while (1) {
 		st[cn][b] = st1[c];
 		b++;
 		c++;
 		st[cn][b] = 0;
 		if (st1[c] == 0) break;
 		if (st1[c] == ' ') {
 			cn++;
 			b = 0;
 			c++;
 		}
 	}
 	cn++;
 	for (a = 0; a < cn; a++) {
 		for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
 		if (b == words) b = -1;
 		bi[a] = b;
 		sprintf(stringBuffer, "\n<pre>Word: \"%s\"  Position in vocabulary: %lld</pre>\n", st[a], bi[a]);
 		if (b == -1) {
 			sprintf(stringBuffer+strlen(stringBuffer), "Out of dictionary word!\n");
 			break;
 		}
 	}
 	if (b == -1) goto end;
 	for (a = 0; a < size; a++) vec[a] = 0;
 	for (b = 0; b < cn; b++) {
 		if (bi[b] == -1) continue;
 		for (a = 0; a < size; a++) vec[a] += M[a + bi[b] * size];
 	}
 	len = 0;
 	for (a = 0; a < size; a++) len += vec[a] * vec[a];
 	len = sqrt(len);
 	for (a = 0; a < size; a++) vec[a] /= len;
 	for (a = 0; a < N; a++) bestd[a] = -1;
 	for (a = 0; a < N; a++) bestw[a][0] = 0;
 	for (c = 0; c < words; c++) {
 		a = 0;
 		for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
 		if (a == 1) continue;
 		dist = 0;
 		for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
 		for (a = 0; a < N; a++) {
 			if (dist > bestd[a]) {
 				for (d = N - 1; d > a; d--) {
 					bestd[d] = bestd[d - 1];
 					strcpy(bestw[d], bestw[d - 1]);
 				}
 				bestd[a] = dist;
 				besti[a] = c;
 				strcpy(bestw[a], &vocab[c * max_w]);
 				break;
 			}
 		}
 	}
 	AV* array = newAV();
 	for (a = 0; a < N; a++) {
 		HV* hash = newHV();
 		hv_store(hash, "word", strlen("word"), newSVpvf(bestw[a], 0), 0);
 		hv_store(hash, "dist", strlen("dist"), newSVnv(bestd[a]), 0);
 		AV *vector = newAV();
 		for (b = 0; b < size; b++) {
 			av_push(vector, newSVnv(M[b + besti[a] * size]));
 		}
 		hv_store(hash, "vector", strlen("vector"), newRV_noinc((SV*)vector), 0);
 		av_push(array, newRV_noinc((SV*)hash));
 	}
  end:
 	return newRV_noinc((SV*)array);
 }


 __DATA__

 @@ index.html.ep
 <!DOCTYPE html>
 <html>
 <head>
 	<title>DeReKo-Word-Vector-Distances</title>
 	<script src="http://code.jquery.com/jquery-latest.min.js"></script>
 	<script src="//d3js.org/d3.v3.min.js" charset="utf-8"></script>
 	<script src="http://klinux10/word2vec/tsne.js"></script>
 <style>
 svg {
 //  border: 1px solid #333;
 //  margin-right: 5px;
 //  margin-bottom: 5px;
 }
 #wrapper {
     width: 100%;
 //   border: 1px solid red;
     overflow: hidden; /* will contain if #first is longer than #second */
 }
 #first {
  width: 300px;
  margin-right: 20px;
     float:left; /* add this */
 //    border: 1px solid green;
 }
 #second {
     border: 1px solid #333;
     overflow: hidden; /* if you don't want #second to wrap below #first */
 }
 </style>
 <script>

 var opt = {epsilon: 1, perplexity: 8};
 var T = new tsnejs.tSNE(opt); // create a tSNE instance

 var Y;

 var data;

 function updateEmbedding() {
   var Y = T.getSolution();
   svg.selectAll('.u')
     .data(data.words)
     .attr("transform", function(d, i) { return "translate(" +
                                           ((Y[i][0]*20*ss + tx) + 400) + "," +
                                           ((Y[i][1]*20*ss + ty) + 400) + ")"; });
 }

 var svg;
 function drawEmbedding() {
     $("#embed").empty();
     var div = d3.select("#embed");

     // get min and max in each column of Y
     var Y = T.Y;

     svg = div.append("svg") // svg is global
     .attr("width", 800)
     .attr("height", 800);

     var g = svg.selectAll(".b")
       .data(data.words)
       .enter().append("g")
       .attr("class", "u");

     g.append("text")
       .attr("text-anchor", "top")
       .attr("font-size", 12)
       .attr("fill", "#333")
       .text(function(d) { return d; });

     var zoomListener = d3.behavior.zoom()
       .scaleExtent([0.1, 10])
       .center([0,0])
       .on("zoom", zoomHandler);
     zoomListener(svg);
 }

 var tx=0, ty=0;
 var ss=1;
 var iter_id=-1;

 function zoomHandler() {
   tx = d3.event.translate[0];
   ty = d3.event.translate[1];
   ss = d3.event.scale;
   updateEmbedding();
 }

 var stepnum = 0;

 function stopStep() {
   clearInterval(iter_id);
 }

 function step() {
   var i = T.iter;
   if(i >= 1000) {
    stopStep();
   } else {
     var cost = T.step(); // do a few steps
     $("#cost").html("iteration " + i + ", cost: " + cost);
     updateEmbedding();
   }
 }

 	function showMap(j) {
 		data=j;
     T.iter=0;
     T.initDataRaw(data.vecs); // init embedding
     drawEmbedding(); // draw initial embedding

 			if(iter_id >= 0) {
 				clearInterval(iter_id);
 			}
     //T.debugGrad();
     iter_id = setInterval(step, 1);
     //step();
 	}

 $(window).xxload(function() {
   $.getJSON( "http://klinux10/word2vec/dings.json", function( j ) {
     data = j;
     T.initDataRaw(data.vecs); // init embedding
     drawEmbedding(); // draw initial embedding

     // T.debugGrad();
     iter_id = setInterval(step, 1);
     // step();

   });
 });

 </script>
 </head>
 <body>
 	<p>Word vector model based on  DeReKo-2015-II. Trained with <a href="https://code.google.com/p/word2vec/">word2vec</a> using the following parameters:</p>
   <pre>
 -cbow 1 -size 300 -window 7 -negative 5 -hs 0 -sample 1e-5 -threads 44 -binary 1 -iter 5
 </pre>
 </p>
   <form action="<%=url_for('/')->to_abs%>" method="GET">
     Word: <input type="text" name="word" value="<%= $word %>">
     Neighbours: <input type="text" name="n" value="<%= $no_nbs %>">
     <input type="submit" value="Show neighbours">
   </form>
   <br>
 	% if($list) {
   <h3>Nearest neighbours of "<%= $word %>"</h3>
 <div id="wrapper">
     <table id="first">
 		<tr>
      <th align="right">Pos.</th><th align="left">Word</th><th align="right">Cosine dist.</th>
    </tr>
 	% my $i=1; my @words; my @vecs; for my $item (@$list) {
   % push @vecs, $item->{vector};
   % push @words, $item->{word};
 		<tr>
      <td align="right">
   		<%= $i++ %>.
      </td>
      <td>
   		<a href="/?word=<%= $item->{word} %>">
 				<%= $item->{word} %>
 			</a>
      </td>
      <td align="right">
   		<%= sprintf("%.3f", $item->{dist}) %>
      </td>
 		</tr>
 	% }
   </table>
 		<script>
 			% use Mojo::ByteStream 'b';
 $(window).load(function() {
 			showMap(<%= b(Mojo::JSON::to_json({words => \@words, vecs => \@vecs})); %>);
 });
     </script>
 	% }
 <div id="second" style="width:800px; height:800px; font-family: arial;">
 <div id="embed"></div>
 <div id="cost" style="text-align:left; font-family: Impact;"></div>
   </div>
   </div>
   </body>
 </html>
	#!/usr/local/bin/perl
	use Inline C;
	use Mojolicious::Lite;
	use Mojo::JSON qw(decode_json encode_json to_json);
	use Encode qw(decode encode);
	use Mojo::Server::Daemon;

	# -cbow 1 -size 200 -window 8 -negative 25 -hs 0 -sample 1e-4 -threads 40 -binary 1 -iter 15
	init_net("vectors14.bin");

	get '/' => sub {
	my $c = shift;
	my $word=$c->param('word');
	my $no_nbs=$c->param('n') \|\| 50;
	my $list;
	if(defined($word) && $word !~ /^\s*$/) {
	$c->inactivity_timeout(300);
	$c->app->log->debug('Looking for neighbours of '.$word);
	$list = get_neighbours(encode("iso-8859-1", $word), $no_nbs);
	}
	$c->render(template=>"index", word=>$word, no_nbs=>$no_nbs, list=> $list);
	};

	app->start;

	exit;

	__END__

	__C__
	#include <stdio.h>
	#include <string.h>
	#include <math.h>
	#include <malloc.h>
	#include <stdlib.h> //strlen

	#define max_size 2000
	#define max_w 50
	#define MAX_NEIGHBOURS 1000

	//the thread function
	void connection_handler(void );

	char *bestw[MAX_NEIGHBOURS];
	char file_name[max_size], st[100][max_size];
	float dist, len, bestd[MAX_NEIGHBOURS], vec[max_size];
	long long words, size, a, b, c, d, cn, bi[100], besti[MAX_NEIGHBOURS];
	char ch;
	float *M;
	char *vocab;
	char *stringBuffer;

	int init_net(char *file_name) {
	FILE *f;

	stringBuffer = malloc(64000);
	f = fopen(file_name, "rb");
	if (f == NULL) {
	printf("Input file %s not found\n", file_name);
	return -1;
	}
	fscanf(f, "%lld", &words);
	fscanf(f, "%lld", &size);
	vocab = (char )malloc((long long)words max_w * sizeof(char));
	for (a = 0; a < MAX_NEIGHBOURS; a++) bestw[a] = (char )malloc(max_size sizeof(char));
	M = (float )malloc((long long)words (long long)size * sizeof(float));
	if (M == NULL) {
	printf("Cannot allocate memory: %lld MB %lld %lld\n", (long long)words * size * sizeof(float) / 1048576, words, size);
	return -1;
	}
	for (b = 0; b < words; b++) {
	a = 0;
	while (1) {
	vocab[b * max_w + a] = fgetc(f);
	if (feof(f) \|\| (vocab[b * max_w + a] == ' ')) break;
	if ((a < max_w) && (vocab[b * max_w + a] != '\n')) a++;
	}
	vocab[b * max_w + a] = 0;
	fread(&M[b * size], sizeof(float), size, f);
	len = 0;
	for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
	len = sqrt(len);
	for (a = 0; a < size; a++) M[a + b * size] /= len;
	}
	fclose(f);
	return 0;
	}

	SV get_neighbours(char st1, int N) {
	if(N>MAX_NEIGHBOURS) N=MAX_NEIGHBOURS;

	FILE *out=stdout;
	*stringBuffer=0;

	for (a = 0; a < N; a++) bestd[a] = 0;
	for (a = 0; a < N; a++) bestw[a][0] = 0;
	a = 0;
	cn = 0;
	b = 0;
	c = 0;
	while (1) {
	st[cn][b] = st1[c];
	b++;
	c++;
	st[cn][b] = 0;
	if (st1[c] == 0) break;
	if (st1[c] == ' ') {
	cn++;
	b = 0;
	c++;
	}
	}
	cn++;
	for (a = 0; a < cn; a++) {
	for (b = 0; b < words; b++) if (!strcmp(&vocab[b * max_w], st[a])) break;
	if (b == words) b = -1;
	bi[a] = b;
	sprintf(stringBuffer, "\n<pre>Word: \"%s\" Position in vocabulary: %lld</pre>\n", st[a], bi[a]);
	if (b == -1) {
	sprintf(stringBuffer+strlen(stringBuffer), "Out of dictionary word!\n");
	break;
	}
	}
	if (b == -1) goto end;
	for (a = 0; a < size; a++) vec[a] = 0;
	for (b = 0; b < cn; b++) {
	if (bi[b] == -1) continue;
	for (a = 0; a < size; a++) vec[a] += M[a + bi[b] * size];
	}
	len = 0;
	for (a = 0; a < size; a++) len += vec[a] * vec[a];
	len = sqrt(len);
	for (a = 0; a < size; a++) vec[a] /= len;
	for (a = 0; a < N; a++) bestd[a] = -1;
	for (a = 0; a < N; a++) bestw[a][0] = 0;
	for (c = 0; c < words; c++) {
	a = 0;
	for (b = 0; b < cn; b++) if (bi[b] == c) a = 1;
	if (a == 1) continue;
	dist = 0;
	for (a = 0; a < size; a++) dist += vec[a] * M[a + c * size];
	for (a = 0; a < N; a++) {
	if (dist > bestd[a]) {
	for (d = N - 1; d > a; d--) {
	bestd[d] = bestd[d - 1];
	strcpy(bestw[d], bestw[d - 1]);
	}
	bestd[a] = dist;
	besti[a] = c;
	strcpy(bestw[a], &vocab[c * max_w]);
	break;
	}
	}
	}
	AV* array = newAV();
	for (a = 0; a < N; a++) {
	HV* hash = newHV();
	hv_store(hash, "word", strlen("word"), newSVpvf(bestw[a], 0), 0);
	hv_store(hash, "dist", strlen("dist"), newSVnv(bestd[a]), 0);
	AV *vector = newAV();
	for (b = 0; b < size; b++) {
	av_push(vector, newSVnv(M[b + besti[a] * size]));
	}
	hv_store(hash, "vector", strlen("vector"), newRV_noinc((SV*)vector), 0);
	av_push(array, newRV_noinc((SV*)hash));
	}
	end:
	return newRV_noinc((SV*)array);
	}


	__DATA__

	@@ index.html.ep
	<!DOCTYPE html>
	<html>
	<head>
	<title>DeReKo-Word-Vector-Distances</title>
	<script src="http://code.jquery.com/jquery-latest.min.js"></script>
	<script src="//d3js.org/d3.v3.min.js" charset="utf-8"></script>
	<script src="http://klinux10/word2vec/tsne.js"></script>
	<style>
	svg {
	// border: 1px solid #333;
	// margin-right: 5px;
	// margin-bottom: 5px;
	}
	#wrapper {
	width: 100%;
	// border: 1px solid red;
	overflow: hidden; /* will contain if #first is longer than #second */
	}
	#first {
	width: 300px;
	margin-right: 20px;
	float:left; /* add this */
	// border: 1px solid green;
	}
	#second {
	border: 1px solid #333;
	overflow: hidden; /* if you don't want #second to wrap below #first */
	}
	</style>
	<script>

	var opt = {epsilon: 1, perplexity: 8};
	var T = new tsnejs.tSNE(opt); // create a tSNE instance

	var Y;

	var data;

	function updateEmbedding() {
	var Y = T.getSolution();
	svg.selectAll('.u')
	.data(data.words)
	.attr("transform", function(d, i) { return "translate(" +
	((Y[i][0]20ss + tx) + 400) + "," +
	((Y[i][1]20ss + ty) + 400) + ")"; });
	}

	var svg;
	function drawEmbedding() {
	$("#embed").empty();
	var div = d3.select("#embed");

	// get min and max in each column of Y
	var Y = T.Y;

	svg = div.append("svg") // svg is global
	.attr("width", 800)
	.attr("height", 800);

	var g = svg.selectAll(".b")
	.data(data.words)
	.enter().append("g")
	.attr("class", "u");

	g.append("text")
	.attr("text-anchor", "top")
	.attr("font-size", 12)
	.attr("fill", "#333")
	.text(function(d) { return d; });

	var zoomListener = d3.behavior.zoom()
	.scaleExtent([0.1, 10])
	.center([0,0])
	.on("zoom", zoomHandler);
	zoomListener(svg);
	}

	var tx=0, ty=0;
	var ss=1;
	var iter_id=-1;

	function zoomHandler() {
	tx = d3.event.translate[0];
	ty = d3.event.translate[1];
	ss = d3.event.scale;
	updateEmbedding();
	}

	var stepnum = 0;

	function stopStep() {
	clearInterval(iter_id);
	}

	function step() {
	var i = T.iter;
	if(i >= 1000) {
	stopStep();
	} else {
	var cost = T.step(); // do a few steps
	$("#cost").html("iteration " + i + ", cost: " + cost);
	updateEmbedding();
	}
	}

	function showMap(j) {
	data=j;
	T.iter=0;
	T.initDataRaw(data.vecs); // init embedding
	drawEmbedding(); // draw initial embedding

	if(iter_id >= 0) {
	clearInterval(iter_id);
	}
	//T.debugGrad();
	iter_id = setInterval(step, 1);
	//step();
	}

	$(window).xxload(function() {
	$.getJSON( "http://klinux10/word2vec/dings.json", function( j ) {
	data = j;
	T.initDataRaw(data.vecs); // init embedding
	drawEmbedding(); // draw initial embedding

	// T.debugGrad();
	iter_id = setInterval(step, 1);
	// step();

	});
	});

	</script>
	</head>
	<body>
	<p>Word vector model based on DeReKo-2015-II. Trained with <a href="https://code.google.com/p/word2vec/">word2vec</a> using the following parameters:</p>
	<pre>
	-cbow 1 -size 300 -window 7 -negative 5 -hs 0 -sample 1e-5 -threads 44 -binary 1 -iter 5
	</pre>
	</p>
	<form action="<%=url_for('/')->to_abs%>" method="GET">
	Word: <input type="text" name="word" value="<%= $word %>">
	Neighbours: <input type="text" name="n" value="<%= $no_nbs %>">
	<input type="submit" value="Show neighbours">
	</form>
	<br>
	% if($list) {
	<h3>Nearest neighbours of "<%= $word %>"</h3>
	<div id="wrapper">
	<table id="first">
	<tr>
	<th align="right">Pos.</th><th align="left">Word</th><th align="right">Cosine dist.</th>
	</tr>
	% my $i=1; my @words; my @vecs; for my $item (@$list) {
	% push @vecs, $item->{vector};
	% push @words, $item->{word};
	<tr>
	<td align="right">
	<%= $i++ %>.
	</td>
	<td>
	<a href="/?word=<%= $item->{word} %>">
	<%= $item->{word} %>
	</a>
	</td>
	<td align="right">
	<%= sprintf("%.3f", $item->{dist}) %>
	</td>
	</tr>
	% }
	</table>
	<script>
	% use Mojo::ByteStream 'b';
	$(window).load(function() {
	showMap(<%= b(Mojo::JSON::to_json({words => \@words, vecs => \@vecs})); %>);
	});
	</script>
	% }
	<div id="second" style="width:800px; height:800px; font-family: arial;">
	<div id="embed"></div>
	<div id="cost" style="text-align:left; font-family: Impact;"></div>
	</div>
	</div>
	</body>
	</html>