Fix collocator lookups for merged models and guard word ranks
w2v.merge was unusable: the server died at startup with "Undefined subroutine
&main::mergeVectors", because mergeVectors was not exported.
With that fixed, count based collocators and precomputed profiles were looked
up with unshifted ranks. The collocator db and the profiles belong to the
primary model, which mergeVectors puts at [merged_end, words), so its ranks
have to be shifted back. The adjustment existed but was dead code, as it read
$IDS::DeReKoVecs::Read::mergedEnd while the value was assigned to
$main::mergedEnd. Instead of wiring a second copy of the value, the merge
offset is now read from the C layer, which is the only place that knows it.
Out of range ranks reach libcollocatordb straight from the request and crash
the process there, so a single GET /getClassicCollocators?w=<beyond vocabulary>
was enough to kill a worker. Ranks are validated before use now, both for the
collocator db and for the model matrix in getVecs, and out of range ids yield
an empty result.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: I26ef508e3682cbe00df27d4de5588d1f1493c701
diff --git a/lib/IDS/DeReKoVecs/Read.pm b/lib/IDS/DeReKoVecs/Read.pm
index 0da0cf4..f75ca90 100644
--- a/lib/IDS/DeReKoVecs/Read.pm
+++ b/lib/IDS/DeReKoVecs/Read.pm
@@ -7,7 +7,6 @@
my $src_file = undef;
-our $mergedEnd=0;
our $opt_p = 5676;
our $opt_C;
@@ -31,7 +30,7 @@
use Mojo::Cache;
use Exporter qw(import);
-our @EXPORT = qw(init_net load_sprofiles getVocabSize getDowntimeCalendar getCollocationAssociation getClassicCollocatorsCached getSimilarProfiles getSimilarProfilesCached getBiggestMergedDifferences filter_garbage get_neighbours getVecs getWordNumber dump_vecs dump_for_numpy cos_similarity_as_json get_version getPosWiseW2VCollocators configure_cache cache_stats);
+our @EXPORT = qw(init_net load_sprofiles mergeVectors getVocabSize getDowntimeCalendar getCollocationAssociation getClassicCollocatorsCached getSimilarProfiles getSimilarProfilesCached getBiggestMergedDifferences getMergedEnd filter_garbage get_neighbours getVecs getWordNumber dump_vecs dump_for_numpy cos_similarity_as_json get_version getPosWiseW2VCollocators configure_cache cache_stats);
my $cccache = Mojo::Cache->new(max_keys => $CC_CACHE_MAX_KEYS); # classic collocator cache
my $spcache = Mojo::Cache->new(max_keys => $SP_CACHE_MAX_KEYS); # similar profile cache
@@ -80,8 +79,12 @@
sub getClassicCollocatorsCached {
my ($c, $word, $compare_to) = @_;
my $s2 = "";
- if($word > $mergedEnd) {
- $word-=$mergedEnd;
+ # The collocator db is the one of the primary model, which sits at
+ # [merged_end, words) in the merged vocabulary, so its ranks have to be
+ # shifted back before they can be looked up.
+ my $merged_end = getMergedEnd();
+ if($word >= $merged_end) {
+ $word -= $merged_end;
}
my $pipe;
@@ -133,6 +136,10 @@
sub getSimilarProfilesCached {
my ($c, $word) = @_;
+ # like the collocator db, the precomputed profiles belong to the primary
+ # model, see getClassicCollocatorsCached()
+ my $merged_end = getMergedEnd();
+ $word -= $merged_end if $word >= $merged_end;
my $profiles = $opt_C ? undef : $spcache->get($word);
if(!defined $profiles) {
$profiles = getSimilarProfiles($word);