Compute logDice as defined by Rychly, and split off the auto focus score
ca_logdice() and ca_dice() multiplied the node frequency by the window
size, while the citation right above ca_logdice() and the tooltip in the
DeReKoVecs web UI both give Rychly's (2008) definition,
14 + log2(2*f12/(f1+f2)).
The window size does not belong there. It is correct in the expected
frequency of the other measures, where f1 * window_size counts the
positions a collocate can occupy, but the Dice coefficient relates the
co-occurrence frequency to how often the two words occur at all, so its
denominator sums marginal frequencies. Adding f1 * window_size, a count
of window positions, to f2, a count of word tokens, mixes units, and it
made the coefficient asymmetric: swapping node and collocate changed the
score, 0.64 against 3.96 for Grund and triftiger from the same counts.
Because f1 * window_size dominated the denominator for a frequent node,
rare collocates were penalised. Among the collocates of "Grund" in a 5+5
window, "triftiger" scored 0.64, below "Berlin" at 2.03, although Berlin
co-occurs with Grund less often than chance predicts. They are now 3.96
and 3.84, and comparable with other tools such as Sketch Engine.
The auto focus score, reported as LDaf, keeps that factor and moves into
its own function, ca_focus_score(). It is Dice-like but not logDice, and
the two are deliberately not on a common scale: the penalty on wide
windows is what makes it sensitive to how concentrated a pair is, which
is what tells actual collocations from words that merely share contexts,
and is why LDaf orders collocates well enough to be the default order.
Its values are unchanged, bit-identical for all 504 collocates of the
five nodes checked against the state before this change, as are the
selected windows.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: I72bd69d4d2a74127ca87bf20d94f9ffafda6c191
diff --git a/src/collocatordb.cc b/src/collocatordb.cc
index f327c4b..4da2e06 100644
--- a/src/collocatordb.cc
+++ b/src/collocatordb.cc
@@ -191,9 +191,15 @@
(o22 > 0 ? o22 * log(o22 / e22) : 0)));
}
+// The Dice coefficient relates the co-occurrence frequency to how often the two
+// words occur at all, so its denominator sums marginal word frequencies. It
+// therefore takes no window size factor, unlike the scores above, which need one
+// in their expected frequency: w1 * window_size would count window positions
+// rather than word tokens, and adding that to w2 would also make the coefficient
+// asymmetric.
static double ca_dice(uint64_t w1, uint64_t w2, uint64_t w12, uint64_t n,
uint64_t window_size) {
- double r1 = (double)w1 * window_size, c1 = w2;
+ double r1 = (double)w1, c1 = w2;
return 2 * w12 / (c1 + r1);
}
@@ -203,6 +209,25 @@
// Advances in Slavonic Natural Language Processing, RASLAN, 6–9.
static double ca_logdice(uint64_t w1, uint64_t w2, uint64_t w12,
uint64_t n, uint64_t window_size) {
+ double r1 = (double)w1, c1 = w2;
+ return 14 + log2(2 * w12 / (c1 + r1));
+}
+
+// The auto focus score, reported as LDaf: its maximum over all selections of
+// positions both picks the auto focus window and is the value shown for it. It
+// used to be computed by ca_logdice(), which is why that one carried a window
+// size factor.
+//
+// It is Dice-like, but it is not logDice and the two are not on a common scale:
+// multiplying f1 by the number of positions penalizes wide windows. That
+// penalty is the point. It makes the score sensitive to how concentrated a pair
+// is, which is what tells actual collocations from words that merely share
+// contexts, and is why LDaf orders collocates more usefully than the other
+// measures. logDice itself cannot serve here: its denominator does not depend
+// on the window while the co-occurrence count only grows as positions are
+// added, so the widest window would always win.
+static double ca_focus_score(uint64_t w1, uint64_t w2, uint64_t w12, uint64_t n,
+ uint64_t window_size) {
double r1 = (double)w1 * window_size, c1 = w2;
return 14 + log2(2 * w12 / (c1 + r1));
}
@@ -776,8 +801,13 @@
md_nws = ca_md(f1, f2, sum, total, 2 * WINDOW_SIZE),
ld = ca_logdice(f1, f2, sum, total, true_window_size);
+ // LDaf is the auto focus score itself, i.e. the highest value that any
+ // selection of positions reaches. It is deliberately not logDice of the
+ // selected window: the width penalty is what makes the score sensitive to
+ // how concentrated a pair is, and that sensitivity is why LDaf orders
+ // collocates more usefully than the other measures.
int bestWindow = usedPositions;
- double bestAF = ld;
+ double bestFocus = ca_focus_score(f1, f2, sum, total, true_window_size);
// if(f1<75000000)
// #pragma omp parallel for reduction(max:bestAF)
// #pragma omp target teams distribute parallel for reduction(max:bestAF)
@@ -792,10 +822,10 @@
if (((1 << pos) & bitmask & usedPositions) != 0)
currentWindowSum += sumWindow[pos];
}
- double currentAF = ca_logdice(f1, f2, currentWindowSum, total,
- __builtin_popcount(bitmask));
- if (currentAF > bestAF) {
- bestAF = currentAF;
+ double currentAF = ca_focus_score(f1, f2, currentWindowSum, total,
+ __builtin_popcount(bitmask));
+ if (currentAF > bestFocus) {
+ bestFocus = currentAF;
bestWindow = bitmask;
}
}
@@ -815,7 +845,7 @@
ca_pmi(f1, f2, sumWindow[WINDOW_SIZE - 1], total, 1),
ca_dice(f1, f2, sum, total, true_window_size),
ld,
- bestAF,
+ bestFocus,
usedPositions,
bestWindow};
}
diff --git a/tests/basic_test.c b/tests/basic_test.c
index 75eae5b..53e2736 100644
--- a/tests/basic_test.c
+++ b/tests/basic_test.c
@@ -28,7 +28,7 @@
void test_collocation_scores() {
COLLOCATORDB* cdb = open_collocatordb(dbpath);
TEST_ASSERT(cdb != NULL);
- char *expected = " { \"f1\": 217,\"w1\":\"Aluminium\", \"N\": 152743, \"collocates\": [{\"word\":\"Anwendungstechnologie\",\"f2\":16,\"f\":16,\"npmi\":0.594849,\"pmi\":8.4592,\"llr\":188.227,\"lfmd\":16.4592,\"md\":12.4592,\"md_nws\":10.1373,\"dice\":0.0711111,\"ld\":10.1862,\"ln_count\":16,\"rn_count\":0,\"ln_pmi\":9.4592,\"rn_pmi\":-1,\"ldaf\":11.1358,\"win\":32,\"afwin\":32}]}\n";
+ char *expected = " { \"f1\": 217,\"w1\":\"Aluminium\", \"N\": 152743, \"collocates\": [{\"word\":\"Anwendungstechnologie\",\"f2\":16,\"f\":16,\"npmi\":0.594849,\"pmi\":8.4592,\"llr\":188.227,\"lfmd\":16.4592,\"md\":12.4592,\"md_nws\":10.1373,\"dice\":0.137339,\"ld\":11.1358,\"ln_count\":16,\"rn_count\":0,\"ln_pmi\":9.4592,\"rn_pmi\":-1,\"ldaf\":11.1358,\"win\":32,\"afwin\":32}]}\n";
char *produced = get_collocation_scores_as_json(cdb, 62, 966);
TEST_CHECK(strcmp(produced, expected) == 0);
TEST_MSG("Expected: %s", expected);
@@ -40,7 +40,7 @@
COLLOCATORDB* cdb = open_collocatordb(dbpath);
TEST_ASSERT(cdb != NULL);
char *json = get_collocators_as_json(cdb, testword);
- char *needle = "\"word\":\"um\",\"f2\":264,\"f\":5,\"npmi\":-0.0556349,\"pmi\":-0.958074,\"llr\":2.87723,\"lfmd\":3.68578,\"md\":1.36385,\"md_nws\":0.363854,\"dice\":0.00169952,\"ld\":4.79935,\"ln_count\":0,\"rn_count\":1,\"ln_pmi\":-1,\"rn_pmi\":-1,\"ldaf\":4.79935,\"win\":668,\"afwin\":668";
+ char *needle = "\"word\":\"um\",\"f2\":264,\"f\":5,\"npmi\":-0.0556349,\"pmi\":-0.958074,\"llr\":2.87723,\"lfmd\":3.68578,\"md\":1.36385,\"md_nws\":0.363854,\"dice\":0.00720461,\"ld\":6.88314,\"ln_count\":0,\"rn_count\":1,\"ln_pmi\":-1,\"rn_pmi\":-1,\"ldaf\":4.79935,\"win\":668,\"afwin\":668";
TEST_CHECK(strstr(json, needle) > 0);
TEST_MSG("Expected to contain: %s", needle);
}