CA: flag imputed cells in multi-VC comparisons
When a collocate is attested in one virtual corpus but not in another,
add_multi_vc_comparisons() imputes the missing scores from a pooled floor.
The resulting delta/winner/loser columns then describe presence vs. absence
rather than a measured contrast, and nothing in the output said so.
Record the missingness pattern before imputation and expose it as
`imputed_<label>` (per label), `n_imputed` (count) and `imputed` (any), so
that `filter(!imputed)` keeps only collocates attested in every compared
virtual corpus.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Change-Id: I25fb0196f1b75519c126130fc19bc62d40675e17
diff --git a/R/collocationAnalysis.R b/R/collocationAnalysis.R
index 7240eca..41eb8d2 100644
--- a/R/collocationAnalysis.R
+++ b/R/collocationAnalysis.R
@@ -60,6 +60,7 @@
#' \item Ranks per label/measure with the pattern \code{rank_<label>_<measure>} (1 is best) and the corresponding percentile ranks \code{percentile_rank_<label>_<measure>}.
#' \item Pairwise contrasts for two-label comparisons, e.g. \code{delta_<measure>}, \code{delta_rank_<measure>}, and \code{delta_percentile_rank_<measure>}.
#' \item Summary columns describing the strongest labels per measure (\code{winner_*}, \code{runner_up_*}, \code{loser_*}, and \code{max_delta_*}), including winner/loser \code{webUIRequestUrl} columns. In multi-VC comparisons, missing per-label concordance URLs are derived from another available row URL for the same \code{node}/\code{collocate} by replacing the \code{cq} parameter with the target label's virtual corpus. Unsuffixed \code{winner_webUIRequestUrl} and \code{loser_webUIRequestUrl} columns are populated only when the score-based URL choices agree.
+#' \item \code{imputed_<label>}, \code{n_imputed}, and \code{imputed}: flags marking rows whose scores were not observed for some label but imputed (see \code{missingScoreQuantile}). Filter with \code{dplyr::filter(!imputed)} to keep only collocates attested in every compared virtual corpus.
#' \item Optional helper columns such as \code{query}, \code{example}, or \code{url} when example retrieval is requested.
#' }
#' @importFrom dplyr arrange desc slice_head bind_rows group_by mutate ungroup left_join select row_number all_of first
@@ -726,6 +727,18 @@
comparison <- dplyr::left_join(comparison, rank_data, by = c("node", "collocate"))
+ # Record which label/measure cells are absent *before* any imputation happens below.
+ # Deltas computed from imputed cells reflect presence/absence of the collocate in a
+ # virtual corpus, not a measured contrast, so users need to be able to tell them apart.
+ imputed_flags <- lapply(labels, function(safe_label) {
+ label_score_cols <- intersect(paste0(score_cols, "_", safe_label), names(comparison))
+ if (length(label_score_cols) == 0) {
+ return(rep(FALSE, nrow(comparison)))
+ }
+ Reduce(`|`, lapply(label_score_cols, function(col) is.na(comparison[[col]])))
+ })
+ names(imputed_flags) <- paste0("imputed_", labels)
+
rank_replacements <- numeric(0)
rank_column_names <- grep("^rank_", names(comparison), value = TRUE)
if (length(rank_column_names) > 0) {
@@ -1230,6 +1243,16 @@
comparison[[max_delta_pct_col]] <- max_deltas
}
+ for (flag_col in names(imputed_flags)) {
+ comparison[[flag_col]] <- imputed_flags[[flag_col]]
+ }
+ if (length(imputed_flags) > 0) {
+ comparison$n_imputed <- as.integer(Reduce(`+`, lapply(imputed_flags, as.integer)))
+ } else {
+ comparison$n_imputed <- rep(0L, nrow(comparison))
+ }
+ comparison$imputed <- comparison$n_imputed > 0L
+
collapse_consensus_url_columns <- function(url_cols) {
if (length(url_cols) == 0) {
return(rep(NA_character_, nrow(comparison)))
diff --git a/man/collocationAnalysis-KorAPConnection-method.Rd b/man/collocationAnalysis-KorAPConnection-method.Rd
index 8a3c899..e7875f4 100644
--- a/man/collocationAnalysis-KorAPConnection-method.Rd
+++ b/man/collocationAnalysis-KorAPConnection-method.Rd
@@ -99,6 +99,7 @@
\item Ranks per label/measure with the pattern \code{rank_<label>_<measure>} (1 is best) and the corresponding percentile ranks \code{percentile_rank_<label>_<measure>}.
\item Pairwise contrasts for two-label comparisons, e.g. \code{delta_<measure>}, \code{delta_rank_<measure>}, and \code{delta_percentile_rank_<measure>}.
\item Summary columns describing the strongest labels per measure (\code{winner_*}, \code{runner_up_*}, \code{loser_*}, and \code{max_delta_*}), including winner/loser \code{webUIRequestUrl} columns. In multi-VC comparisons, missing per-label concordance URLs are derived from another available row URL for the same \code{node}/\code{collocate} by replacing the \code{cq} parameter with the target label's virtual corpus. Unsuffixed \code{winner_webUIRequestUrl} and \code{loser_webUIRequestUrl} columns are populated only when the score-based URL choices agree.
+\item \code{imputed_<label>}, \code{n_imputed}, and \code{imputed}: flags marking rows whose scores were not observed for some label but imputed (see \code{missingScoreQuantile}). Filter with \code{dplyr::filter(!imputed)} to keep only collocates attested in every compared virtual corpus.
\item Optional helper columns such as \code{query}, \code{example}, or \code{url} when example retrieval is requested.
}
}
diff --git a/tests/testthat/test-collocations.R b/tests/testthat/test-collocations.R
index 94bc453..183de99 100644
--- a/tests/testthat/test-collocations.R
+++ b/tests/testthat/test-collocations.R
@@ -322,6 +322,81 @@
)
})
+test_that("add_multi_vc_comparisons flags imputed cells", {
+ sample_result <- tibble::tibble(
+ node = c("n", "n", "n"),
+ collocate = c("c1", "c2", "c2"),
+ vc = c("corpusSigle=/A/", "corpusSigle=/A/", "corpusSigle=/B/"),
+ label = c("A", "A", "B"),
+ N = c(100, 100, 100),
+ O = c(10, 10, 20),
+ O1 = c(50, 50, 50),
+ O2 = c(30, 30, 30),
+ E = c(5, 5, 5),
+ w = c(2, 2, 2),
+ leftContextSize = c(1, 1, 1),
+ rightContextSize = c(1, 1, 1),
+ frequency = c(10, 10, 20),
+ webUIRequestUrl = c(
+ "https://korap.example/A1",
+ "https://korap.example/A2",
+ "https://korap.example/B2"
+ ),
+ logDice = c(6, 5, 7),
+ pmi = c(3, 2, 4)
+ )
+
+ enriched <- RKorAPClient:::add_multi_vc_comparisons(sample_result)
+
+ expect_true(all(c("imputed_A", "imputed_B", "n_imputed", "imputed") %in% colnames(enriched)))
+
+ c1 <- enriched[enriched$collocate == "c1", ]
+ c2 <- enriched[enriched$collocate == "c2", ]
+
+ # c1 only occurs in A, so B's scores had to be imputed
+ expect_true(all(c1$imputed_B))
+ expect_true(all(!c1$imputed_A))
+ expect_true(all(c1$imputed))
+ expect_true(all(c1$n_imputed == 1L))
+
+ # c2 occurs in both, so nothing is imputed
+ expect_true(all(!c2$imputed_A))
+ expect_true(all(!c2$imputed_B))
+ expect_true(all(!c2$imputed))
+ expect_true(all(c2$n_imputed == 0L))
+
+ # c1's delta is measured against the imputed floor rather than against observed data:
+ # the absent label loses, at or below the weakest score actually attested anywhere.
+ expect_true(all(is.finite(c1$max_delta_logDice)))
+ expect_equal(unique(c1$loser_logDice), "B")
+ expect_lte(unique(c1$loser_logDice_value), min(sample_result$logDice))
+})
+
+test_that("add_multi_vc_comparisons reports no imputation when all labels are complete", {
+ sample_result <- tibble::tibble(
+ node = rep("n", 4),
+ collocate = c("c1", "c1", "c2", "c2"),
+ vc = rep(c("vc1", "vc2"), 2),
+ label = rep(c("A", "B"), 2),
+ N = rep(100, 4),
+ O = c(10, 20, 30, 40),
+ O1 = rep(50, 4),
+ O2 = rep(30, 4),
+ E = rep(5, 4),
+ w = rep(2, 4),
+ leftContextSize = rep(1, 4),
+ rightContextSize = rep(1, 4),
+ frequency = c(10, 20, 30, 40),
+ logDice = c(5, 7, 6, 4),
+ pmi = c(2, 3, 4, 1)
+ )
+
+ enriched <- RKorAPClient:::add_multi_vc_comparisons(sample_result)
+
+ expect_true(all(!enriched$imputed))
+ expect_true(all(enriched$n_imputed == 0L))
+})
+
test_that("add_multi_vc_comparisons handles more than two labels", {
sample_result <- tibble::tibble(
node = rep("n", 3),