Offer cacheAs on the other query functions as well
frequencyQuery(), corpusStats(), collocationScoreQuery() and
textMetadata() take a cacheAs file too now, not because they are slow -
they are not, next to a collocation analysis running for hours - but
because a file of one's own is what keeps an analysis reproducible. KorAP
corpora grow, so the same query returns different numbers next year, and
the scores computed from them may change with the package; a result kept
next to the script stays what it was, and the script runs without a
server at all.
The parameter is documented as what it is, next to the connection's
cache, which is a transparent speed-up one can throw away.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Change-Id: I9bac8931e7a38a91adcbfe58ef4cb5bcdaaf3c47
diff --git a/NEWS.md b/NEWS.md
index 587eed0..e19d928 100644
--- a/NEWS.md
+++ b/NEWS.md
@@ -1,5 +1,7 @@
# unpublished dev version 1.3.0.9000
+- **`cacheAs` is now offered by `frequencyQuery()`, `corpusStats()`, `collocationScoreQuery()` and `textMetadata()`** as well, not only by `collocationAnalysis()`. It is a different thing from the connection's `cache`, which is a transparent speed-up: a `cacheAs` file belongs to the caller and is what keeps an analysis reproducible, since KorAP corpora grow and the same query returns different numbers next year. That is worth having for the quick functions too, where caching for speed would be pointless
+
- **`cacheAs` files record the version that wrote them** and are refused, with a warning, when that is older than 1.4.0. Their contents are finished results including the association scores, which this version computes differently, so an old file would silently hand back numbers that would not be arrived at again - something no comparison of parameters can notice. The file is then recomputed and overwritten; pass a different name to keep it. How loud a query is no longer counts as a parameter either: `verbose` does not change what is returned
- **the names of a named `vc` vector are now used as labels** by `frequencyQuery()`, `corpusStats()` and `collocationScoreQuery()`, as `collocationAnalysis()` already did. `frequencyQuery()` ignored them, `corpusStats()` put them into row names, which the first `bind_rows()` drops, and `collocationScoreQuery()` derived a label from the corpus definitions instead, so that `c(before = ..., since = ...)` came out as `"1990 & pubDat…"`. Where a vector carries no names, nothing changes: no `label` column appears that was not there before
diff --git a/R/KorAPCorpusStats.R b/R/KorAPCorpusStats.R
index 6fe44d8..0c23304 100644
--- a/R/KorAPCorpusStats.R
+++ b/R/KorAPCorpusStats.R
@@ -34,6 +34,7 @@
#' @param vc string describing the virtual corpus. An empty string (default) means the whole corpus, as far as it is license-wise accessible.
#' @param verbose logical. If `TRUE`, additional diagnostics are printed.
#' @param as.df return result as data frame instead of as S4 object?
+#' @param cacheAs path to an RDS file to keep the result in. If the file exists and records the same call, it is read back instead of contacting the server; otherwise the query is run and its result stored there. Unlike the connection's `cache`, this file belongs to the caller, which is what keeps an analysis reproducible once the corpus has grown or the scores have changed. Defaults to \code{NULL} (no file).
#' @return Object containing corpus statistics with the following information:
#' \describe{
#' \item{`vc`}{Virtual corpus definition used (empty string for entire corpus)}
@@ -73,8 +74,19 @@
setMethod("corpusStats", "KorAPConnection", function(kco,
vc = "",
verbose = kco@verbose,
- as.df = FALSE) {
- if (length(vc) > 1) {
+ as.df = FALSE,
+ cacheAs = NULL) {
+ cacheRecord <- NULL
+ if (!is.null(cacheAs)) {
+ cacheAs <- cacheAsFileName(cacheAs)
+ cacheRecord <- cacheAsRecord(environment(), NULL, kco)
+ cached <- readCacheAs(cacheAs, kco, cacheRecord, "corpus statistics")
+ if (!is.null(cached)) {
+ return(cached)
+ }
+ }
+
+ stats <- if (length(vc) > 1) {
# the names of a named vc vector would end up as row names, which the first
# bind_rows() drops, so they are kept as a column instead
vcLabel <- vcLabels(vc)
@@ -186,6 +198,11 @@
)
}
}
+
+ if (!is.null(cacheAs)) {
+ writeCacheAs(cacheAs, kco, cacheRecord, "corpus statistics", stats)
+ }
+ stats
})
#' @rdname KorAPCorpusStats-class
diff --git a/R/KorAPQuery.R b/R/KorAPQuery.R
index 77774c8..a68f03e 100644
--- a/R/KorAPQuery.R
+++ b/R/KorAPQuery.R
@@ -1382,6 +1382,7 @@
#' @param vc virtual corpus definition(s) (can be a vector)
#' @param conf.level confidence level of the returned confidence interval (passed through [ci()] to [prop.test()]).
#' @param as.alternatives LOGICAL that specifies if the query terms should be treated as alternatives. If `as.alternatives` is TRUE, the sum over all query hits, instead of the respective vc token sizes is used as total for the calculation of relative frequencies.
+#' @param cacheAs path to an RDS file to keep the result in. If the file exists and records the same call, it is read back instead of contacting the server; otherwise the query is run and its result stored there. Unlike the connection's `cache`, this file belongs to the caller, which is what keeps an analysis reproducible once the corpus has grown or the scores have changed. Defaults to \code{NULL} (no file).
#' @param ... further arguments passed to or from other methods (see [corpusQuery()]), most notably `expand`, a logical that decides if `query` and `vc` parameters are expanded to all of their combinations. It defaults to `TRUE`, if `query` and `vc` have different lengths, and to `FALSE` otherwise.
#' @export
#'
@@ -1397,8 +1398,19 @@
setMethod(
"frequencyQuery", "KorAPConnection",
- function(kco, query, vc = "", conf.level = 0.95, as.alternatives = FALSE, ...) {
- (if (as.alternatives) {
+ function(kco, query, vc = "", conf.level = 0.95, as.alternatives = FALSE,
+ cacheAs = NULL, ...) {
+ cacheRecord <- NULL
+ if (!is.null(cacheAs)) {
+ cacheAs <- cacheAsFileName(cacheAs)
+ cacheRecord <- cacheAsRecord(environment(), list(...), kco)
+ cached <- readCacheAs(cacheAs, kco, cacheRecord, "frequency query")
+ if (!is.null(cached)) {
+ return(cached)
+ }
+ }
+
+ result <- (if (as.alternatives) {
corpusQuery(kco, query, vc, metadataOnly = TRUE, as.df = TRUE, ...) |>
group_by(vc) |>
mutate(total = sum(totalResults))
@@ -1407,6 +1419,11 @@
mutate(total = corpusStats(kco, vc = vc, as.df = TRUE)$tokens)
}) |>
ci(conf.level = conf.level)
+
+ if (!is.null(cacheAs)) {
+ writeCacheAs(cacheAs, kco, cacheRecord, "frequency query", result)
+ }
+ result
}
)
diff --git a/R/collocationScoreQuery.R b/R/collocationScoreQuery.R
index 6f28bc4..8458583 100644
--- a/R/collocationScoreQuery.R
+++ b/R/collocationScoreQuery.R
@@ -25,6 +25,7 @@
#' @param observed if collocation frequencies are already known (or estimated from a sample) they can be passed as a vector here, otherwise: NA
#' @param ignoreCollocateCase logical, set to TRUE if collocate case should be ignored
#' @param withinSpan KorAP span specification (see <https://korap.ids-mannheim.de/doc/ql/poliqarp-plus?embedded=true#spans>) for collocations to be searched within. Defaults to `base/s=s`.
+#' @param cacheAs path to an RDS file to keep the result in. If the file exists and records the same call, it is read back instead of contacting the server; otherwise the query is run and its result stored there. Unlike the connection's `cache`, this file belongs to the caller, which is what keeps an analysis reproducible once the corpus has grown or the scores have changed. Defaults to \code{NULL} (no file).
#'
#' @return tibble with query KorAP web request URL, all observed values and association scores
#'
@@ -71,8 +72,19 @@
smoothingConstant = .5,
observed = NA,
ignoreCollocateCase = FALSE,
- withinSpan = "base/s=s"
+ withinSpan = "base/s=s",
+ cacheAs = NULL
) {
+ cacheRecord <- NULL
+ if (!is.null(cacheAs)) {
+ cacheAs <- cacheAsFileName(cacheAs)
+ cacheRecord <- cacheAsRecord(environment(), NULL, kco)
+ cached <- readCacheAs(cacheAs, kco, cacheRecord, "collocation scores")
+ if (!is.null(cached)) {
+ return(cached)
+ }
+ }
+
# https://stackoverflow.com/questions/8096313/no-visible-binding-for-global-variable-note-in-r-cmd-check
O1 <- O2 <- O <- N <- E <- w <- 0
@@ -122,7 +134,7 @@
NULL
}
- tibble(
+ result <- tibble(
node = node,
collocate = combinations$collocate,
# the names the caller gave their virtual corpora, where there
@@ -158,6 +170,10 @@
) %>%
mutate(!!! lapply(scoreFunctions, mapply, .$O1, .$O2, .$O, .$N, .$E, .$w))
+ if (!is.null(cacheAs)) {
+ writeCacheAs(cacheAs, kco, cacheRecord, "collocation scores", result)
+ }
+ result
})
# #' @export
diff --git a/R/textMetadata.R b/R/textMetadata.R
index a0b115a..182bf97 100644
--- a/R/textMetadata.R
+++ b/R/textMetadata.R
@@ -15,6 +15,7 @@
#' @param kco [KorAPConnection()] object (obtained e.g. from `KorAPConnection()`)
#' @param textSigle unique text id (concatenation of corpus, document and text ids, separated by `/`, e.g. ) or vector thereof
#' @param verbose logical. If `TRUE`, additional diagnostics are printed. Defaults to `kco@verbose`.
+#' @param cacheAs path to an RDS file to keep the result in. If the file exists and records the same call, it is read back instead of contacting the server; otherwise the query is run and its result stored there. Unlike the connection's `cache`, this file belongs to the caller, which is what keeps an analysis reproducible once the corpus has grown or the scores have changed. Defaults to \code{NULL} (no file).
#'
#' @return Tibble with columns for each metadata property. In case of errors, such as non-existing texts/sigles, the tibble will also contain a column called `errors`.
#' If there are metadata columns you cannot make sense of, please ignore them. The function simply returns all the metadata it gets from the server.
@@ -32,10 +33,20 @@
#' @export
setMethod(
"textMetadata", "KorAPConnection",
- function(kco, textSigle, verbose = kco@verbose) {
+ function(kco, textSigle, verbose = kco@verbose, cacheAs = NULL) {
+ cacheRecord <- NULL
+ if (!is.null(cacheAs)) {
+ cacheAs <- cacheAsFileName(cacheAs)
+ cacheRecord <- cacheAsRecord(environment(), NULL, kco)
+ cached <- readCacheAs(cacheAs, kco, cacheRecord, "text metadata")
+ if (!is.null(cached)) {
+ return(cached)
+ }
+ }
+
# https://stackoverflow.com/questions/8096313/no-visible-binding-for-global-variable-note-in-r-cmd-check
key <- 0
- if (length(textSigle) > 1) {
+ metadata <- if (length(textSigle) > 1) {
do.call(bind_rows, Map(function(atomicSigle) {
textMetadata(kco, atomicSigle)
}, textSigle))
@@ -78,5 +89,10 @@
}
res
}
+
+ if (!is.null(cacheAs)) {
+ writeCacheAs(cacheAs, kco, cacheRecord, "text metadata", metadata)
+ }
+ metadata
}
)
diff --git a/man/collocationScoreQuery-KorAPConnection-method.Rd b/man/collocationScoreQuery-KorAPConnection-method.Rd
index b7ff06a..b6314fa 100644
--- a/man/collocationScoreQuery-KorAPConnection-method.Rd
+++ b/man/collocationScoreQuery-KorAPConnection-method.Rd
@@ -18,7 +18,8 @@
smoothingConstant = 0.5,
observed = NA,
ignoreCollocateCase = FALSE,
- withinSpan = "base/s=s"
+ withinSpan = "base/s=s",
+ cacheAs = NULL
)
}
\arguments{
@@ -47,6 +48,8 @@
\item{ignoreCollocateCase}{logical, set to TRUE if collocate case should be ignored}
\item{withinSpan}{KorAP span specification (see \url{https://korap.ids-mannheim.de/doc/ql/poliqarp-plus?embedded=true#spans}) for collocations to be searched within. Defaults to \code{base/s=s}.}
+
+\item{cacheAs}{path to an RDS file to keep the result in. If the file exists and records the same call, it is read back instead of contacting the server; otherwise the query is run and its result stored there. Unlike the connection's \code{cache}, this file belongs to the caller, which is what keeps an analysis reproducible once the corpus has grown or the scores have changed. Defaults to \code{NULL} (no file).}
}
\value{
tibble with query KorAP web request URL, all observed values and association scores
diff --git a/man/corpusStats-KorAPConnection-method.Rd b/man/corpusStats-KorAPConnection-method.Rd
index 74ba851..7e576ce 100644
--- a/man/corpusStats-KorAPConnection-method.Rd
+++ b/man/corpusStats-KorAPConnection-method.Rd
@@ -5,7 +5,7 @@
\alias{corpusStats}
\title{Get corpus size and statistics}
\usage{
-\S4method{corpusStats}{KorAPConnection}(kco, vc = "", verbose = kco@verbose, as.df = FALSE)
+\S4method{corpusStats}{KorAPConnection}(kco, vc = "", verbose = kco@verbose, as.df = FALSE, cacheAs = NULL)
}
\arguments{
\item{kco}{\code{\link[=KorAPConnection]{KorAPConnection()}} object (obtained e.g. from \code{KorAPConnection()}}
@@ -15,6 +15,8 @@
\item{verbose}{logical. If \code{TRUE}, additional diagnostics are printed.}
\item{as.df}{return result as data frame instead of as S4 object?}
+
+\item{cacheAs}{path to an RDS file to keep the result in. If the file exists and records the same call, it is read back instead of contacting the server; otherwise the query is run and its result stored there. Unlike the connection's \code{cache}, this file belongs to the caller, which is what keeps an analysis reproducible once the corpus has grown or the scores have changed. Defaults to \code{NULL} (no file).}
}
\value{
Object containing corpus statistics with the following information:
diff --git a/man/frequencyQuery-KorAPConnection-method.Rd b/man/frequencyQuery-KorAPConnection-method.Rd
index cc9c361..5ae060a 100644
--- a/man/frequencyQuery-KorAPConnection-method.Rd
+++ b/man/frequencyQuery-KorAPConnection-method.Rd
@@ -11,6 +11,7 @@
vc = "",
conf.level = 0.95,
as.alternatives = FALSE,
+ cacheAs = NULL,
...
)
}
@@ -25,6 +26,8 @@
\item{as.alternatives}{LOGICAL that specifies if the query terms should be treated as alternatives. If \code{as.alternatives} is TRUE, the sum over all query hits, instead of the respective vc token sizes is used as total for the calculation of relative frequencies.}
+\item{cacheAs}{path to an RDS file to keep the result in. If the file exists and records the same call, it is read back instead of contacting the server; otherwise the query is run and its result stored there. Unlike the connection's \code{cache}, this file belongs to the caller, which is what keeps an analysis reproducible once the corpus has grown or the scores have changed. Defaults to \code{NULL} (no file).}
+
\item{...}{further arguments passed to or from other methods (see \code{\link[=corpusQuery]{corpusQuery()}}), most notably \code{expand}, a logical that decides if \code{query} and \code{vc} parameters are expanded to all of their combinations. It defaults to \code{TRUE}, if \code{query} and \code{vc} have different lengths, and to \code{FALSE} otherwise.}
}
\value{
diff --git a/man/textMetadata-KorAPConnection-method.Rd b/man/textMetadata-KorAPConnection-method.Rd
index aadbee3..5f1f462 100644
--- a/man/textMetadata-KorAPConnection-method.Rd
+++ b/man/textMetadata-KorAPConnection-method.Rd
@@ -5,7 +5,7 @@
\alias{textMetadata}
\title{Retrieve metadata for a text, identified by its sigle (id)}
\usage{
-\S4method{textMetadata}{KorAPConnection}(kco, textSigle, verbose = kco@verbose)
+\S4method{textMetadata}{KorAPConnection}(kco, textSigle, verbose = kco@verbose, cacheAs = NULL)
}
\arguments{
\item{kco}{\code{\link[=KorAPConnection]{KorAPConnection()}} object (obtained e.g. from \code{KorAPConnection()})}
@@ -13,6 +13,8 @@
\item{textSigle}{unique text id (concatenation of corpus, document and text ids, separated by \code{/}, e.g. ) or vector thereof}
\item{verbose}{logical. If \code{TRUE}, additional diagnostics are printed. Defaults to \code{kco@verbose}.}
+
+\item{cacheAs}{path to an RDS file to keep the result in. If the file exists and records the same call, it is read back instead of contacting the server; otherwise the query is run and its result stored there. Unlike the connection's \code{cache}, this file belongs to the caller, which is what keeps an analysis reproducible once the corpus has grown or the scores have changed. Defaults to \code{NULL} (no file).}
}
\value{
Tibble with columns for each metadata property. In case of errors, such as non-existing texts/sigles, the tibble will also contain a column called \code{errors}.
diff --git a/tests/testthat/test-cache-as.R b/tests/testthat/test-cache-as.R
new file mode 100644
index 0000000..2a15b75
--- /dev/null
+++ b/tests/testthat/test-cache-as.R
@@ -0,0 +1,92 @@
+test_that("cacheAsFileName appends .rds where it is missing", {
+ expect_equal(RKorAPClient:::cacheAsFileName("analysis"), "analysis.rds")
+ expect_equal(RKorAPClient:::cacheAsFileName("analysis.rds"), "analysis.rds")
+ expect_equal(RKorAPClient:::cacheAsFileName("analysis.RDS"), "analysis.RDS")
+})
+
+test_that("a cache file is written and read back, without contacting the server", {
+ skip_if_offline()
+ kco <- KorAPConnection(accessToken = NULL, verbose = FALSE)
+ dir <- file.path(tempdir(), "cacheAsTest")
+ dir.create(dir, showWarnings = FALSE)
+ on.exit(unlink(dir, recursive = TRUE), add = TRUE)
+
+ # every query function that takes cacheAs, so that one added later without a
+ # cache file of its own does not go unnoticed
+ queries <- list(
+ "frequencyQuery" = function(f) frequencyQuery(kco, "Ameisenplage", cacheAs = f),
+ "corpusStats" = function(f) corpusStats(kco, vc = "pubDate since 2020", as.df = TRUE, cacheAs = f),
+ "collocationScoreQuery" = function(f) collocationScoreQuery(kco, "Grund", "triftiger", cacheAs = f),
+ "textMetadata" = function(f) textMetadata(kco, "WPD17/L79/98721", cacheAs = f)
+ )
+
+ for (name in names(queries)) {
+ query <- queries[[name]]
+ file <- file.path(dir, name)
+
+ fresh <- query(file)
+ expect_true(file.exists(paste0(file, ".rds")), info = name)
+
+ fromCache <- local({
+ testthat::local_mocked_bindings(
+ apiCall = function(...) stop("server must not be contacted"),
+ .package = "RKorAPClient"
+ )
+ query(file)
+ })
+ expect_equal(fromCache, fresh, info = name)
+
+ # what produced the result lives in the file only, not in the value
+ expect_null(attr(fromCache, RKorAPClient:::cacheAsAttribute), info = name)
+ expect_equal(
+ attr(readRDS(paste0(file, ".rds")), RKorAPClient:::cacheAsAttribute)$scoreVersion,
+ RKorAPClient:::cacheAsScoreVersion,
+ info = name
+ )
+ }
+})
+
+test_that("a cache file written before the scores were corrected is refused", {
+ skip_if_offline()
+ kco <- KorAPConnection(accessToken = NULL, verbose = FALSE)
+ file <- tempfile(fileext = ".rds")
+ on.exit(unlink(file), add = TRUE)
+
+ frequencyQuery(kco, "Ameisenplage", cacheAs = file)
+ aged <- readRDS(file)
+ record <- attr(aged, RKorAPClient:::cacheAsAttribute)
+ # as a file from before the corrections has it: an older score generation, and
+ # an older version to go with it
+ record$scoreVersion <- "1.3.0"
+ record$packageVersion <- "1.3.0"
+ attr(aged, RKorAPClient:::cacheAsAttribute) <- record
+ saveRDS(aged, file)
+
+ expect_warning(frequencyQuery(kco, "Ameisenplage", cacheAs = file), "1\\.3\\.0")
+ # and the refused file is replaced by a current one, so the warning comes once
+ expect_silent(frequencyQuery(kco, "Ameisenplage", cacheAs = file))
+})
+
+test_that("a query that differs is recomputed rather than read back", {
+ skip_if_offline()
+ kco <- KorAPConnection(accessToken = NULL, verbose = FALSE)
+ file <- tempfile(fileext = ".rds")
+ on.exit(unlink(file), add = TRUE)
+
+ frequencyQuery(kco, "Ameisenplage", cacheAs = file)
+ expect_warning(frequencyQuery(kco, "Heuschreckenplage", cacheAs = file), "query")
+})
+
+test_that("verbosity is not part of what a cache file records", {
+ skip_if_offline()
+ kco <- KorAPConnection(accessToken = NULL, verbose = FALSE)
+ file <- tempfile(fileext = ".rds")
+ on.exit(unlink(file), add = TRUE)
+
+ corpusStats(kco, vc = "pubDate since 2020", as.df = TRUE, cacheAs = file)
+ # how loud a query is does not change what it returns, so it must not make
+ # the file be thrown away
+ expect_silent(
+ corpusStats(kco, vc = "pubDate since 2020", as.df = TRUE, verbose = TRUE, cacheAs = file)
+ )
+})