Write the memory mappable model form after training
derekovecs memory maps <model>.vecs and <model>.words next to the model. When
they are missing it builds them on its first start, which takes a while and
needs write access to the directory the models live in. A server in a
container usually has that directory mounted read only and then does not start
at all:
Converting /models/dereko-2026-i.vecs to memory mappable structures
Cannot open /models/dereko-2026-i.vecs.words or .../dereko-2026-i.vecs.vecs
dereko2vec writes both files right after saving the vectors now, and
vecs2mmap does it for models that were trained earlier.
The conversion mirrors init_net() of derekovecs including the order of the
floating point operations. Verified against the model of the derekovecs test
data: the vectors come out byte identical, and derekovecs serves the same
neighbours and collocators from the generated files, with the model directory
mounted read only and without converting anything itself.
Words are cut off at 49 characters plus the terminator instead of filling all
50 bytes, so that every entry is terminated within its own record.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: I297e2db98bd425aa388e441aca6d2b0b14f0c66c
diff --git a/src/dereko2vec.c b/src/dereko2vec.c
index 767956a..9b10c22 100644
--- a/src/dereko2vec.c
+++ b/src/dereko2vec.c
@@ -21,6 +21,8 @@
#include <pthread.h>
#include <collocatordb.h>
+#include "mmap_vecs.h"
+
#define MAX_STRING 100
#define EXP_TABLE_SIZE 1000
#define MAX_EXP 6
@@ -1906,6 +1908,19 @@
free(cl);
}
fclose(fo);
+ /* Write the memory mappable form right away. Otherwise derekovecs builds it
+ on its first start, which takes a while and needs write access to the
+ directory the models live in. The k-means classes are something else, so
+ only word vectors are converted. */
+ if (classes == 0) {
+ if (debug_mode > 0) {
+ printf("Converting %s to memory mappable structures\n", output_file);
+ fflush(stdout);
+ }
+ if (convert_vecs_to_mmap(output_file) != 0)
+ fprintf(stderr, "Could not write the memory mappable form of %s\n",
+ output_file);
+ }
if (save_net_file[0] != 0)
SaveNet();
}
diff --git a/src/mmap_vecs.h b/src/mmap_vecs.h
new file mode 100644
index 0000000..9ad13d7
--- /dev/null
+++ b/src/mmap_vecs.h
@@ -0,0 +1,108 @@
+/* Writes the memory mappable form of a model next to the model file:
+ *
+ * <model>.vecs words * size floats, every word vector normalized to length 1
+ * <model>.words words * MMAP_MAX_W bytes, the word, NUL terminated and padded
+ *
+ * derekovecs memory maps these two files. When they are missing it builds them
+ * itself on its first start, which takes a while and needs write access to the
+ * directory the models live in - which a server does not necessarily have.
+ *
+ * The conversion deliberately mirrors init_net() of derekovecs, including the
+ * order of the floating point operations, so that both produce the same files.
+ */
+
+#ifndef MMAP_VECS_H
+#define MMAP_VECS_H
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* has to match max_w in derekovecs */
+#define MMAP_MAX_W 50
+
+/* Reads the model written by SaveVectors() and writes its memory mappable
+ * form. Returns 0 on success. */
+static int convert_vecs_to_mmap(const char *file_name) {
+ FILE *f, *binvecs, *binwords;
+ char binvecs_fname[4096], binwords_fname[4096];
+ long long words, size, a, b;
+ float len, *M;
+ char *vocab;
+ double val;
+ int is_text = (strstr(file_name, ".txt") != NULL);
+
+ snprintf(binvecs_fname, sizeof(binvecs_fname), "%s.vecs", file_name);
+ snprintf(binwords_fname, sizeof(binwords_fname), "%s.words", file_name);
+
+ if ((f = fopen(file_name, "rb")) == NULL) {
+ fprintf(stderr, "Cannot open %s for the memory mappable conversion\n", file_name);
+ return -1;
+ }
+ if (fscanf(f, "%lld", &words) != 1 || fscanf(f, "%lld", &size) != 1) {
+ fprintf(stderr, "Cannot read the header of %s\n", file_name);
+ fclose(f);
+ return -1;
+ }
+
+ vocab = (char *)calloc((size_t)words * MMAP_MAX_W, sizeof(char));
+ M = (float *)malloc((size_t)words * (size_t)size * sizeof(float));
+ if (vocab == NULL || M == NULL) {
+ fprintf(stderr, "Cannot allocate %lld MB for the memory mappable conversion\n",
+ (long long)((size_t)words * size * sizeof(float) / 1048576));
+ free(vocab);
+ free(M);
+ fclose(f);
+ return -1;
+ }
+
+ for (b = 0; b < words; b++) {
+ a = 0;
+ while (1) {
+ vocab[b * MMAP_MAX_W + a] = fgetc(f);
+ if (feof(f) || (vocab[b * MMAP_MAX_W + a] == ' ')) break;
+ if ((a < MMAP_MAX_W - 1) && (vocab[b * MMAP_MAX_W + a] != '\n')) a++;
+ }
+ vocab[b * MMAP_MAX_W + a] = 0;
+ len = 0;
+ if (is_text) {
+ for (a = 0; a < size; a++) {
+ if (fscanf(f, "%lf", &val) != 1) val = 0;
+ M[a + b * size] = val;
+ len += M[a + b * size] * M[a + b * size];
+ }
+ } else {
+ if (fread(&M[b * size], sizeof(float), size, f) != (size_t)size) {
+ fprintf(stderr, "Unexpected end of %s at word %lld\n", file_name, b);
+ free(vocab);
+ free(M);
+ fclose(f);
+ return -1;
+ }
+ for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
+ }
+ len = sqrt(len);
+ if (len > 0)
+ for (a = 0; a < size; a++) M[a + b * size] /= len;
+ }
+ fclose(f);
+
+ if ((binvecs = fopen(binvecs_fname, "wb")) == NULL ||
+ (binwords = fopen(binwords_fname, "wb")) == NULL) {
+ fprintf(stderr, "Cannot write %s or %s\n", binvecs_fname, binwords_fname);
+ free(vocab);
+ free(M);
+ return -1;
+ }
+ fwrite(M, sizeof(float), (size_t)words * (size_t)size, binvecs);
+ fclose(binvecs);
+ fwrite(vocab, sizeof(char), (size_t)words * MMAP_MAX_W, binwords);
+ fclose(binwords);
+
+ free(vocab);
+ free(M);
+ return 0;
+}
+
+#endif /* MMAP_VECS_H */
diff --git a/src/vecs2mmap.c b/src/vecs2mmap.c
new file mode 100644
index 0000000..3d72355
--- /dev/null
+++ b/src/vecs2mmap.c
@@ -0,0 +1,29 @@
+/* Writes the memory mappable form of an already trained model, so that
+ * derekovecs does not have to build it itself on its first start. dereko2vec
+ * does this right after training, this tool is for models that were trained
+ * before, or whose converted files were lost.
+ */
+
+#include <stdio.h>
+#include <string.h>
+
+#include "mmap_vecs.h"
+
+int main(int argc, char **argv) {
+ int i;
+
+ if (argc < 2 || strcmp(argv[1], "-h") == 0) {
+ printf("usage: %s <model.vecs> [<model.vecs> ...]\n\n"
+ "Writes <model.vecs>.vecs and <model.vecs>.words, the memory\n"
+ "mappable form derekovecs uses.\n", argv[0]);
+ return argc < 2 ? 1 : 0;
+ }
+
+ for (i = 1; i < argc; i++) {
+ printf("Converting %s to memory mappable structures\n", argv[i]);
+ fflush(stdout);
+ if (convert_vecs_to_mmap(argv[i]) != 0)
+ return 1;
+ }
+ return 0;
+}