Fix memory leaks in the request path
Every request leaked the target sum array (cutoff * sizeof(float), i.e. 2 MB
at the default cutoff of 500000), the thread handle array, the target word
list and the collocator results of all window threads. getWordNumber() leaked
a word list on every word lookup, too.
On top of that every string and array returned from C to perl leaked: for a
char* return value Inline::C generates sv_setpv(), which copies the buffer but
never frees it, and for an AV* return value it generates newRV(), which leaves
the array behind with a reference count of one. All functions returning data
to perl therefore return an SV* now and release their buffers.
getClassicCollocators() and getCollocationScores() leaked twice, because
libcollocatordb hands out strdup()ed strings that were strdup()ed again.
Also fixes crashes in the same code paths:
- get_neighbours() divided by zero when called with nosp=1
- getPosWiseW2VCollocators() wrote before the start of its result buffer for
empty results, called malloc(0) for models without a syn layer and
dereferenced a NULL collocator result for multi word input starting with an
out of dictionary word
- getBiggestMergedDifferences() sized its buffer for 50 bytes per record,
which is not enough for long words
- getSimilarProfiles() wrote to buffer[-1] for empty profiles
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: Iff4215745f26156408a056de6d867753993082c2
diff --git a/lib/IDS/DeReKoVecs/derekovecs-server.c b/lib/IDS/DeReKoVecs/derekovecs-server.c
index 5ef2014..e4cc096 100644
--- a/lib/IDS/DeReKoVecs/derekovecs-server.c
+++ b/lib/IDS/DeReKoVecs/derekovecs-server.c
@@ -371,6 +371,14 @@
return syn_nbs;
}
+/* Frees a knn result as returned by getCollocators() via pthread_exit(). */
+void free_knn(knn *nbs) {
+ if (nbs == NULL)
+ return;
+ free(nbs->best);
+ free(nbs);
+}
+
void *getCollocators(void *args) {
knnpars *pars = args;
int N = pars->N;
@@ -500,7 +508,11 @@
return syn1neg_window[target * window_layer_size + window_offset + hidden];
}
-AV *getVecs(AV *array) {
+/* Returns an SV* (not an AV*) on purpose: for an AV* return value Inline::C
+ generates newRV(), which leaves the array itself with a reference count of
+ one after the mortal reference is gone, i.e. it leaks the whole array on
+ every call. */
+SV *getVecs(AV *array) {
int i, b;
AV *result = newAV();
for (i = 0; i <= av_len(array); i++) {
@@ -514,10 +526,13 @@
av_push(result, newRV_noinc((SV *)vector));
}
}
- return result;
+ return newRV_noinc((SV *)result);
}
-char *getSimilarProfiles(long node) {
+/* All functions handing a string back to perl return an SV*, because for a
+ char* return value Inline::C only copies the string into the return SV and
+ never frees the buffer we allocated here. */
+SV *getSimilarProfiles(long node) {
int i;
char buffer[120000];
char pair_buffer[2048];
@@ -525,7 +540,7 @@
buffer[1] = 0;
if (node >= sprofiles_qty) {
printf("Not available in precomputed profile\n");
- return (strdup("[{\"w\":\"not available\", \"v\":0}]\n"));
+ return newSVpv("[{\"w\":\"not available\", \"v\":0}]\n", 0);
}
printf("******* %s ******\n", &vocab[max_w * node]);
@@ -534,19 +549,27 @@
sprintf(pair_buffer, "{\"w\":\"%s\", \"v\":%f},", &vocab[max_w * (sprofiles[node].nbr[i].index)], sprofiles[node].nbr[i].value);
strcat(buffer, pair_buffer);
}
- buffer[strlen(buffer) - 1] = ']';
+ if (i > 0)
+ buffer[strlen(buffer) - 1] = ']';
+ else
+ strcat(buffer, "]");
strcat(buffer, "\n");
printf("%s", buffer);
- return (strdup(buffer));
+ return newSVpv(buffer, 0);
}
-char *getCollocationScores(long node, long collocate) {
- char *res = (cdb ? strdup(get_collocation_scores_as_json(cdb, node, collocate)) : "[]");
- return res;
+/* get_collocat*_as_json() hand out strdup()ed buffers that we own. */
+SV *getCollocationScores(long node, long collocate) {
+ char *json = (cdb ? (char *)get_collocation_scores_as_json(cdb, node, collocate) : NULL);
+ SV *res = newSVpv(json ? json : "[]", 0);
+ free(json);
+ return res;
}
-char *getClassicCollocators(long node) {
- char *res = (cdb ? strdup(get_collocators_as_json(cdb, node)) : "[]");
+SV *getClassicCollocators(long node) {
+ char *json = (cdb ? (char *)get_collocators_as_json(cdb, node) : NULL);
+ SV *res = newSVpv(json ? json : "[]", 0);
+ free(json);
return res;
}
@@ -590,9 +613,13 @@
long getWordNumber(char *word) {
wordlist *wl = getTargetWords(word, 0);
+ long res = 0;
+ if (wl == NULL)
+ return(0);
if(wl->length > 0)
- return(wl->wordi[0]);
- return(0);
+ res = wl->wordi[0];
+ free(wl);
+ return(res);
}
float get_distance(long b, long c) {
@@ -602,7 +629,9 @@
return dist;
}
-char *getBiggestMergedDifferences() {
+/* The result is computed once and then kept in a static buffer for the
+ lifetime of the process. */
+SV *getBiggestMergedDifferences() {
static char *result = NULL;
float dist;
long long a, c;
@@ -612,7 +641,7 @@
result = "[]";
if (result != NULL)
- return result;
+ return newSVpv(result, 0);
printf("Looking for biggest distances between main and merged vectors ...\n");
collocator *best;
@@ -640,7 +669,7 @@
}
}
- result = malloc(N * max_w);
+ result = malloc(N * (max_w + 64));
char *p = (char *) result;
*p++ = '[';
*p = 0;
@@ -648,7 +677,8 @@
p += sprintf(p, "{\"rank\":%lld,\"word\":\"%s\",\"dist\":%.3f},", a, &vocab[best[a].wordi * max_w], 1 - best[a].activation);
}
*--p = ']';
- return (result);
+ free(best);
+ return newSVpv(result, 0);
}
float cos_similarity(long b, long c) {
@@ -658,19 +688,22 @@
return dist;
}
-char *cos_similarity_as_json(char *w1, char *w2) {
+SV *cos_similarity_as_json(char *w1, char *w2) {
wordlist *a, *b;
float res;
+ char json[32];
a = getTargetWords(w1, 0);
b = getTargetWords(w2, 0);
if (a == NULL || b == NULL || a->length != 1 || b->length != 1)
res = -1;
- else
+ else {
res = cos_similarity(a->wordi[0], b->wordi[0]);
- fprintf(stderr, "a: %lld b: %lld res:%f\n", a->wordi[0], b->wordi[0], res);
- char *json = malloc(16);
+ fprintf(stderr, "a: %lld b: %lld res:%f\n", a->wordi[0], b->wordi[0], res);
+ }
+ free(a);
+ free(b);
sprintf(json, "%.5f", res);
- return json;
+ return newSVpv(json, 0);
}
void *_get_neighbours(void *arg) {
@@ -747,28 +780,30 @@
return (fa > fb) - (fa < fb);
}
-char *getPosWiseW2VCollocators(char *word, long maxPerPos, long cutoff, float threshold, const char *format) {
- HV *result = newHV();
+SV *getPosWiseW2VCollocators(char *word, long maxPerPos, long cutoff, float threshold, const char *format) {
float *target_sums = NULL;
- long a, b;
- knn *para_nbs[MAX_THREADS];
+ long a, b, entries = 0;
knn *syn_nbs[MAX_THREADS];
knnpars pars[MAX_THREADS];
- pthread_t *pt = (pthread_t *)malloc((num_threads + 1) * sizeof(pthread_t));
- wordlist *wl;
+ pthread_t *pt = NULL;
+ wordlist *wl = NULL;
int syn_threads = (M2 ? window * 2 : 0);
int search_backw = 0;
- collocator *best = NULL;
- posix_memalign((void **)&best, 128, 10 * (maxPerPos >= 200 ? maxPerPos : 200) * sizeof(collocator));
- memset(best, 0, (maxPerPos >= 200 ? maxPerPos : 200) * sizeof(collocator));
+ char *result = NULL;
+ SV *res_sv;
+
+ for (a = 0; a < MAX_THREADS; a++) syn_nbs[a] = NULL;
if (cutoff < 1 || cutoff > words)
cutoff = words;
wl = getTargetWords(word, search_backw);
- if (wl == NULL || wl->length < 1)
- return "";
+ if (wl == NULL || wl->length < 1 || wl->wordi[0] < 0 || syn_threads < 1) {
+ free(wl);
+ return newSVpv("", 0);
+ }
+ pt = (pthread_t *)malloc((num_threads + 1) * sizeof(pthread_t));
a = posix_memalign((void **)&target_sums, 128, cutoff * sizeof(float));
memset(target_sums, 0, cutoff * sizeof(float));
@@ -780,6 +815,7 @@
pars[a].window_sums = window_sums;
pars[a].wl = wl;
pars[a].N = maxPerPos;
+ pars[a].best = NULL; /* getCollocators() allocates its own result array */
pars[a].threshold = threshold;
pars[a].from = a;
pars[a].upto = a + 1;
@@ -790,34 +826,45 @@
for (a = 0; a < syn_threads; a++) pthread_join(pt[a], (void *)&syn_nbs[a]);
printf("Syn threads joint\n");
fflush(stdout);
- result = malloc(maxPerPos * 80 * syn_threads);
+ result = malloc((maxPerPos > 0 ? maxPerPos : 1) * (max_w + 96) * syn_threads + 16);
char *p = (char *) result;
*p = 0;
if (strcmp(format, "tsv") == 0) {
for (a = syn_threads - 1; a >= 0; a--) {
- for (b = 0; b < syn_nbs[a]->length; b++) {
+ if (syn_nbs[a] == NULL) continue;
+ for (b = 0; b < syn_nbs[a]->length; b++, entries++) {
p += sprintf(p, "%ld\t%s\t%f\n", syn_nbs[a]->best[b].position, &vocab[syn_nbs[a]->best[b].wordi * max_w], syn_nbs[a]->best[b].activation);
}
}
} else {
p += sprintf(p, "[");
for (a = syn_threads - 1; a >= 0; a--) {
- for (b = 0; b < syn_nbs[a]->length; b++) {
+ if (syn_nbs[a] == NULL) continue;
+ for (b = 0; b < syn_nbs[a]->length; b++, entries++) {
p += sprintf(p, "{\"pos\": %ld, \"word\":\"%s\",\"activation\": %f},\n", syn_nbs[a]->best[b].position, &vocab[syn_nbs[a]->best[b].wordi * max_w], syn_nbs[a]->best[b].activation);
}
}
- p-=2;
+ if (entries > 0)
+ p -= 2; /* drop the trailing ",\n" */
p += sprintf(p, "\n]");
}
- return ((char *)result);
+ res_sv = newSVpv(result, 0);
+
+ free(result);
+ free(target_sums);
+ free(pt);
+ free(wl);
+ for (a = 0; a < syn_threads; a++) free_knn(syn_nbs[a]);
+
+ return res_sv;
}
-char *getPosWiseW2VCollocatorsAsTsv(char *word, long maxPerPos, long cutoff, float threshold) {
+SV *getPosWiseW2VCollocatorsAsTsv(char *word, long maxPerPos, long cutoff, float threshold) {
return getPosWiseW2VCollocators(word, maxPerPos, cutoff, threshold, "tsv");
}
-char *getPosWiseW2VCollocatorsAsJson(char *word, long maxPerPos, long cutoff, float threshold) {
+SV *getPosWiseW2VCollocatorsAsJson(char *word, long maxPerPos, long cutoff, float threshold) {
return getPosWiseW2VCollocators(word, maxPerPos, cutoff, threshold, "json");
}
@@ -829,10 +876,12 @@
knn *syn_nbs[MAX_THREADS];
knnpars pars[MAX_THREADS];
pthread_t *pt = (pthread_t *)malloc((num_threads + 1) * sizeof(pthread_t));
- wordlist *wl;
+ wordlist *wl = NULL;
int syn_threads = (M2 ? window * 2 : 0);
int para_threads = (no_similar_profiles ? 0 : num_threads - syn_threads);
+ for (a = 0; a < MAX_THREADS; a++) para_nbs[a] = syn_nbs[a] = NULL;
+
collocator *best = NULL;
posix_memalign((void **)&best, 128, 10 * (N >= 200 ? N : 200) * sizeof(collocator));
memset(best, 0, (N >= 200 ? N : 200) * sizeof(collocator));
@@ -846,7 +895,7 @@
if (wl == NULL || wl->length < 1)
goto end;
- slice = cutoff / para_threads;
+ slice = (para_threads > 0 ? cutoff / para_threads : cutoff);
a = posix_memalign((void **)&target_sums, 128, cutoff * sizeof(float));
memset(target_sums, 0, cutoff * sizeof(float));
@@ -1157,6 +1206,13 @@
}
end:
free(best);
+ free(target_sums);
+ free(pt);
+ free(wl);
+ for (a = 0; a < MAX_THREADS; a++) {
+ free_knn(para_nbs[a]);
+ free_knn(syn_nbs[a]);
+ }
return newRV_noinc((SV *)result);
}