Fix collocator lookups for merged models and guard word ranks
w2v.merge was unusable: the server died at startup with "Undefined subroutine
&main::mergeVectors", because mergeVectors was not exported.
With that fixed, count based collocators and precomputed profiles were looked
up with unshifted ranks. The collocator db and the profiles belong to the
primary model, which mergeVectors puts at [merged_end, words), so its ranks
have to be shifted back. The adjustment existed but was dead code, as it read
$IDS::DeReKoVecs::Read::mergedEnd while the value was assigned to
$main::mergedEnd. Instead of wiring a second copy of the value, the merge
offset is now read from the C layer, which is the only place that knows it.
Out of range ranks reach libcollocatordb straight from the request and crash
the process there, so a single GET /getClassicCollocators?w=<beyond vocabulary>
was enough to kill a worker. Ranks are validated before use now, both for the
collocator db and for the model matrix in getVecs, and out of range ids yield
an empty result.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: I26ef508e3682cbe00df27d4de5588d1f1493c701
diff --git a/lib/IDS/DeReKoVecs/derekovecs-server.c b/lib/IDS/DeReKoVecs/derekovecs-server.c
index 9d073a4..56e037c 100644
--- a/lib/IDS/DeReKoVecs/derekovecs-server.c
+++ b/lib/IDS/DeReKoVecs/derekovecs-server.c
@@ -526,8 +526,11 @@
if (elem != NULL) {
long j = (long)SvNV(*elem);
AV *vector = newAV();
- for (b = 0; b < size; b++) {
- av_push(vector, newSVnv(M[b + j * size]));
+ /* ranks come from the request, reading outside the model would crash */
+ if (j >= 0 && j < words) {
+ for (b = 0; b < size; b++) {
+ av_push(vector, newSVnv(M[b + j * size]));
+ }
}
av_push(result, newRV_noinc((SV *)vector));
}
@@ -535,6 +538,13 @@
return newRV_noinc((SV *)result);
}
+/* Word ids of the collocator db refer to the primary model, which occupies
+ [0, words - merged_end) of its own vocabulary. libcollocatordb crashes on
+ ids outside that range, and the ids come straight from the request. */
+int valid_cdb_node(long node) {
+ return cdb != NULL && node >= 0 && node < words - merged_end;
+}
+
/* All functions handing a string back to perl return an SV*, because for a
char* return value Inline::C only copies the string into the return SV and
never frees the buffer we allocated here. */
@@ -544,7 +554,7 @@
char pair_buffer[2048];
buffer[0] = '[';
buffer[1] = 0;
- if (node >= sprofiles_qty) {
+ if (node < 0 || node >= sprofiles_qty) {
printf("Not available in precomputed profile\n");
return newSVpv("[{\"w\":\"not available\", \"v\":0}]\n", 0);
}
@@ -566,15 +576,21 @@
/* get_collocat*_as_json() hand out strdup()ed buffers that we own. */
SV *getCollocationScores(long node, long collocate) {
- char *json = (cdb ? (char *)get_collocation_scores_as_json(cdb, node, collocate) : NULL);
- SV *res = newSVpv(json ? json : "[]", 0);
+ char *json = NULL;
+ SV *res;
+ if (valid_cdb_node(node) && valid_cdb_node(collocate))
+ json = (char *)get_collocation_scores_as_json(cdb, node, collocate);
+ res = newSVpv(json ? json : "{\"collocates\":[]}", 0);
free(json);
return res;
}
SV *getClassicCollocators(long node) {
- char *json = (cdb ? (char *)get_collocators_as_json(cdb, node) : NULL);
- SV *res = newSVpv(json ? json : "[]", 0);
+ char *json = NULL;
+ SV *res;
+ if (valid_cdb_node(node))
+ json = (char *)get_collocators_as_json(cdb, node);
+ res = newSVpv(json ? json : "{\"collocates\":[]}", 0);
free(json);
return res;
}
@@ -1279,3 +1295,11 @@
unsigned long getVocabSize() {
return (unsigned long) words;
}
+
+/* First rank of the primary model in the merged vocabulary, 0 if no second
+ model was merged in. mergeVectors() puts the merged in model at ranks
+ [0, merged_end) and the primary model - the one the collocator db belongs
+ to - at [merged_end, words). */
+long getMergedEnd() {
+ return (long) merged_end;
+}