Write the memory mappable model form after training
derekovecs memory maps <model>.vecs and <model>.words next to the model. When
they are missing it builds them on its first start, which takes a while and
needs write access to the directory the models live in. A server in a
container usually has that directory mounted read only and then does not start
at all:
Converting /models/dereko-2026-i.vecs to memory mappable structures
Cannot open /models/dereko-2026-i.vecs.words or .../dereko-2026-i.vecs.vecs
dereko2vec writes both files right after saving the vectors now, and
vecs2mmap does it for models that were trained earlier.
The conversion mirrors init_net() of derekovecs including the order of the
floating point operations. Verified against the model of the derekovecs test
data: the vectors come out byte identical, and derekovecs serves the same
neighbours and collocators from the generated files, with the model directory
mounted read only and without converting anything itself.
Words are cut off at 49 characters plus the terminator instead of filling all
50 bytes, so that every entry is terminated within its own record.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: I297e2db98bd425aa388e441aca6d2b0b14f0c66c
diff --git a/src/mmap_vecs.h b/src/mmap_vecs.h
new file mode 100644
index 0000000..9ad13d7
--- /dev/null
+++ b/src/mmap_vecs.h
@@ -0,0 +1,108 @@
+/* Writes the memory mappable form of a model next to the model file:
+ *
+ * <model>.vecs words * size floats, every word vector normalized to length 1
+ * <model>.words words * MMAP_MAX_W bytes, the word, NUL terminated and padded
+ *
+ * derekovecs memory maps these two files. When they are missing it builds them
+ * itself on its first start, which takes a while and needs write access to the
+ * directory the models live in - which a server does not necessarily have.
+ *
+ * The conversion deliberately mirrors init_net() of derekovecs, including the
+ * order of the floating point operations, so that both produce the same files.
+ */
+
+#ifndef MMAP_VECS_H
+#define MMAP_VECS_H
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+/* has to match max_w in derekovecs */
+#define MMAP_MAX_W 50
+
+/* Reads the model written by SaveVectors() and writes its memory mappable
+ * form. Returns 0 on success. */
+static int convert_vecs_to_mmap(const char *file_name) {
+ FILE *f, *binvecs, *binwords;
+ char binvecs_fname[4096], binwords_fname[4096];
+ long long words, size, a, b;
+ float len, *M;
+ char *vocab;
+ double val;
+ int is_text = (strstr(file_name, ".txt") != NULL);
+
+ snprintf(binvecs_fname, sizeof(binvecs_fname), "%s.vecs", file_name);
+ snprintf(binwords_fname, sizeof(binwords_fname), "%s.words", file_name);
+
+ if ((f = fopen(file_name, "rb")) == NULL) {
+ fprintf(stderr, "Cannot open %s for the memory mappable conversion\n", file_name);
+ return -1;
+ }
+ if (fscanf(f, "%lld", &words) != 1 || fscanf(f, "%lld", &size) != 1) {
+ fprintf(stderr, "Cannot read the header of %s\n", file_name);
+ fclose(f);
+ return -1;
+ }
+
+ vocab = (char *)calloc((size_t)words * MMAP_MAX_W, sizeof(char));
+ M = (float *)malloc((size_t)words * (size_t)size * sizeof(float));
+ if (vocab == NULL || M == NULL) {
+ fprintf(stderr, "Cannot allocate %lld MB for the memory mappable conversion\n",
+ (long long)((size_t)words * size * sizeof(float) / 1048576));
+ free(vocab);
+ free(M);
+ fclose(f);
+ return -1;
+ }
+
+ for (b = 0; b < words; b++) {
+ a = 0;
+ while (1) {
+ vocab[b * MMAP_MAX_W + a] = fgetc(f);
+ if (feof(f) || (vocab[b * MMAP_MAX_W + a] == ' ')) break;
+ if ((a < MMAP_MAX_W - 1) && (vocab[b * MMAP_MAX_W + a] != '\n')) a++;
+ }
+ vocab[b * MMAP_MAX_W + a] = 0;
+ len = 0;
+ if (is_text) {
+ for (a = 0; a < size; a++) {
+ if (fscanf(f, "%lf", &val) != 1) val = 0;
+ M[a + b * size] = val;
+ len += M[a + b * size] * M[a + b * size];
+ }
+ } else {
+ if (fread(&M[b * size], sizeof(float), size, f) != (size_t)size) {
+ fprintf(stderr, "Unexpected end of %s at word %lld\n", file_name, b);
+ free(vocab);
+ free(M);
+ fclose(f);
+ return -1;
+ }
+ for (a = 0; a < size; a++) len += M[a + b * size] * M[a + b * size];
+ }
+ len = sqrt(len);
+ if (len > 0)
+ for (a = 0; a < size; a++) M[a + b * size] /= len;
+ }
+ fclose(f);
+
+ if ((binvecs = fopen(binvecs_fname, "wb")) == NULL ||
+ (binwords = fopen(binwords_fname, "wb")) == NULL) {
+ fprintf(stderr, "Cannot write %s or %s\n", binvecs_fname, binwords_fname);
+ free(vocab);
+ free(M);
+ return -1;
+ }
+ fwrite(M, sizeof(float), (size_t)words * (size_t)size, binvecs);
+ fclose(binvecs);
+ fwrite(vocab, sizeof(char), (size_t)words * MMAP_MAX_W, binwords);
+ fclose(binwords);
+
+ free(vocab);
+ free(M);
+ return 0;
+}
+
+#endif /* MMAP_VECS_H */