Fix collocator lookups for merged models and guard word ranks
w2v.merge was unusable: the server died at startup with "Undefined subroutine
&main::mergeVectors", because mergeVectors was not exported.
With that fixed, count based collocators and precomputed profiles were looked
up with unshifted ranks. The collocator db and the profiles belong to the
primary model, which mergeVectors puts at [merged_end, words), so its ranks
have to be shifted back. The adjustment existed but was dead code, as it read
$IDS::DeReKoVecs::Read::mergedEnd while the value was assigned to
$main::mergedEnd. Instead of wiring a second copy of the value, the merge
offset is now read from the C layer, which is the only place that knows it.
Out of range ranks reach libcollocatordb straight from the request and crash
the process there, so a single GET /getClassicCollocators?w=<beyond vocabulary>
was enough to kill a worker. Ranks are validated before use now, both for the
collocator db and for the model matrix in getVecs, and out of range ids yield
an empty result.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: I26ef508e3682cbe00df27d4de5588d1f1493c701
diff --git a/Changelog.md b/Changelog.md
index 33d14e0..9602eaf 100644
--- a/Changelog.md
+++ b/Changelog.md
@@ -1,5 +1,14 @@
# Changelog
+- fixed a crash on out of range word ranks: a single request with a `w`
+ parameter beyond the vocabulary size, or a `getVecsByRanks` request with such
+ a rank, killed the worker process
+- fixed `w2v.merge`, which died at startup with "Undefined subroutine
+ &main::mergeVectors", and the lookup of count based collocators and
+ precomputed profiles for merged models, which used unshifted ranks
+- fixed `getVecsByRanks`, which answered every request with "Undefined
+ subroutine &main::getVecs", so the collocator SOM never got its vectors, and
+ which was only reachable below the `/derekovecs` prefix
- fixed memory leaks in the request path: per request the server leaked the
target sum array, the thread handles, the target word list and the
collocator results of all window threads, plus every string and array
diff --git a/lib/IDS/DeReKoVecs/Read.pm b/lib/IDS/DeReKoVecs/Read.pm
index 0da0cf4..f75ca90 100644
--- a/lib/IDS/DeReKoVecs/Read.pm
+++ b/lib/IDS/DeReKoVecs/Read.pm
@@ -7,7 +7,6 @@
my $src_file = undef;
-our $mergedEnd=0;
our $opt_p = 5676;
our $opt_C;
@@ -31,7 +30,7 @@
use Mojo::Cache;
use Exporter qw(import);
-our @EXPORT = qw(init_net load_sprofiles getVocabSize getDowntimeCalendar getCollocationAssociation getClassicCollocatorsCached getSimilarProfiles getSimilarProfilesCached getBiggestMergedDifferences filter_garbage get_neighbours getVecs getWordNumber dump_vecs dump_for_numpy cos_similarity_as_json get_version getPosWiseW2VCollocators configure_cache cache_stats);
+our @EXPORT = qw(init_net load_sprofiles mergeVectors getVocabSize getDowntimeCalendar getCollocationAssociation getClassicCollocatorsCached getSimilarProfiles getSimilarProfilesCached getBiggestMergedDifferences getMergedEnd filter_garbage get_neighbours getVecs getWordNumber dump_vecs dump_for_numpy cos_similarity_as_json get_version getPosWiseW2VCollocators configure_cache cache_stats);
my $cccache = Mojo::Cache->new(max_keys => $CC_CACHE_MAX_KEYS); # classic collocator cache
my $spcache = Mojo::Cache->new(max_keys => $SP_CACHE_MAX_KEYS); # similar profile cache
@@ -80,8 +79,12 @@
sub getClassicCollocatorsCached {
my ($c, $word, $compare_to) = @_;
my $s2 = "";
- if($word > $mergedEnd) {
- $word-=$mergedEnd;
+ # The collocator db is the one of the primary model, which sits at
+ # [merged_end, words) in the merged vocabulary, so its ranks have to be
+ # shifted back before they can be looked up.
+ my $merged_end = getMergedEnd();
+ if($word >= $merged_end) {
+ $word -= $merged_end;
}
my $pipe;
@@ -133,6 +136,10 @@
sub getSimilarProfilesCached {
my ($c, $word) = @_;
+ # like the collocator db, the precomputed profiles belong to the primary
+ # model, see getClassicCollocatorsCached()
+ my $merged_end = getMergedEnd();
+ $word -= $merged_end if $word >= $merged_end;
my $profiles = $opt_C ? undef : $spcache->get($word);
if(!defined $profiles) {
$profiles = getSimilarProfiles($word);
diff --git a/lib/IDS/DeReKoVecs/derekovecs-server.c b/lib/IDS/DeReKoVecs/derekovecs-server.c
index 9d073a4..56e037c 100644
--- a/lib/IDS/DeReKoVecs/derekovecs-server.c
+++ b/lib/IDS/DeReKoVecs/derekovecs-server.c
@@ -526,8 +526,11 @@
if (elem != NULL) {
long j = (long)SvNV(*elem);
AV *vector = newAV();
- for (b = 0; b < size; b++) {
- av_push(vector, newSVnv(M[b + j * size]));
+ /* ranks come from the request, reading outside the model would crash */
+ if (j >= 0 && j < words) {
+ for (b = 0; b < size; b++) {
+ av_push(vector, newSVnv(M[b + j * size]));
+ }
}
av_push(result, newRV_noinc((SV *)vector));
}
@@ -535,6 +538,13 @@
return newRV_noinc((SV *)result);
}
+/* Word ids of the collocator db refer to the primary model, which occupies
+ [0, words - merged_end) of its own vocabulary. libcollocatordb crashes on
+ ids outside that range, and the ids come straight from the request. */
+int valid_cdb_node(long node) {
+ return cdb != NULL && node >= 0 && node < words - merged_end;
+}
+
/* All functions handing a string back to perl return an SV*, because for a
char* return value Inline::C only copies the string into the return SV and
never frees the buffer we allocated here. */
@@ -544,7 +554,7 @@
char pair_buffer[2048];
buffer[0] = '[';
buffer[1] = 0;
- if (node >= sprofiles_qty) {
+ if (node < 0 || node >= sprofiles_qty) {
printf("Not available in precomputed profile\n");
return newSVpv("[{\"w\":\"not available\", \"v\":0}]\n", 0);
}
@@ -566,15 +576,21 @@
/* get_collocat*_as_json() hand out strdup()ed buffers that we own. */
SV *getCollocationScores(long node, long collocate) {
- char *json = (cdb ? (char *)get_collocation_scores_as_json(cdb, node, collocate) : NULL);
- SV *res = newSVpv(json ? json : "[]", 0);
+ char *json = NULL;
+ SV *res;
+ if (valid_cdb_node(node) && valid_cdb_node(collocate))
+ json = (char *)get_collocation_scores_as_json(cdb, node, collocate);
+ res = newSVpv(json ? json : "{\"collocates\":[]}", 0);
free(json);
return res;
}
SV *getClassicCollocators(long node) {
- char *json = (cdb ? (char *)get_collocators_as_json(cdb, node) : NULL);
- SV *res = newSVpv(json ? json : "[]", 0);
+ char *json = NULL;
+ SV *res;
+ if (valid_cdb_node(node))
+ json = (char *)get_collocators_as_json(cdb, node);
+ res = newSVpv(json ? json : "{\"collocates\":[]}", 0);
free(json);
return res;
}
@@ -1279,3 +1295,11 @@
unsigned long getVocabSize() {
return (unsigned long) words;
}
+
+/* First rank of the primary model in the merged vocabulary, 0 if no second
+ model was merged in. mergeVectors() puts the merged in model at ranks
+ [0, merged_end) and the primary model - the one the collocator db belongs
+ to - at [merged_end, words). */
+long getMergedEnd() {
+ return (long) merged_end;
+}