Calculate and store statistics for named VC.
Change-Id: Iaed0c72fb9e889cd9eb2453a2e79cabe81293c0f
diff --git a/Changes b/Changes
index 5dd77e3..2d0d3ca 100644
--- a/Changes
+++ b/Changes
@@ -1,3 +1,7 @@
+# version 0.80.1-SNAPSHOT
+
+- Calculate and store statistics for named VC.
+
# version 0.80-SNAPSHOT
- Introduced APIDeprecationFilter (#759)
diff --git a/pom.xml b/pom.xml
index d4eb865..33d862f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -4,7 +4,7 @@
<modelVersion>4.0.0</modelVersion>
<groupId>de.ids-mannheim.korap.kustvakt</groupId>
<artifactId>Kustvakt</artifactId>
- <version>0.80-SNAPSHOT</version>
+ <version>0.80.1-SNAPSHOT</version>
<properties>
<java.version>21</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
diff --git a/src/main/java/de/ids_mannheim/korap/dao/QueryDao.java b/src/main/java/de/ids_mannheim/korap/dao/QueryDao.java
index 9f2f70a..93a75e9 100644
--- a/src/main/java/de/ids_mannheim/korap/dao/QueryDao.java
+++ b/src/main/java/de/ids_mannheim/korap/dao/QueryDao.java
@@ -51,7 +51,7 @@
public int createQuery (String name, ResourceType type, QueryType queryType,
CorpusAccess requiredAccess, String koralQuery, String definition,
String description, String status, boolean isCached,
- String createdBy, String query, String queryLanguage)
+ String createdBy, String query, String queryLanguage, String statistics)
throws KustvaktException {
QueryDO q = new QueryDO();
@@ -67,6 +67,7 @@
q.setCached(isCached);
q.setQuery(query);
q.setQueryLanguage(queryLanguage);
+ q.setStatistics(statistics);
entityManager.persist(q);
return q.getId();
@@ -75,7 +76,8 @@
public void editQuery (QueryDO queryDO, String name, ResourceType type,
CorpusAccess requiredAccess, String koralQuery, String definition,
String description, String status, boolean isCached,
- String queryStr, String queryLanguage) throws KustvaktException {
+ String queryStr, String queryLanguage, String statistics)
+ throws KustvaktException {
if (name != null && !name.isEmpty()) {
queryDO.setName(name);
@@ -104,6 +106,9 @@
if (queryLanguage != null && !queryLanguage.isEmpty()) {
queryDO.setQueryLanguage(queryLanguage);
}
+ if (statistics != null && !statistics.isEmpty()) {
+ queryDO.setStatistics(statistics);
+ }
queryDO.setCached(isCached);
entityManager.merge(queryDO);
}
diff --git a/src/main/java/de/ids_mannheim/korap/entity/QueryDO.java b/src/main/java/de/ids_mannheim/korap/entity/QueryDO.java
index 3a5ebde..e0c2b04 100644
--- a/src/main/java/de/ids_mannheim/korap/entity/QueryDO.java
+++ b/src/main/java/de/ids_mannheim/korap/entity/QueryDO.java
@@ -67,6 +67,8 @@
@OneToMany(mappedBy = "query", fetch = FetchType.LAZY,
cascade = CascadeType.REMOVE)
private List<Role> roles;
+
+ private String statistics;
@Override
public String toString () {
diff --git a/src/main/java/de/ids_mannheim/korap/init/NamedVCLoader.java b/src/main/java/de/ids_mannheim/korap/init/NamedVCLoader.java
index 55c10f6..3be7984 100644
--- a/src/main/java/de/ids_mannheim/korap/init/NamedVCLoader.java
+++ b/src/main/java/de/ids_mannheim/korap/init/NamedVCLoader.java
@@ -159,6 +159,12 @@
// updateVCinDB
storeVCinDB(vcId, json, existingVC, apiVersion);
}
+ else if (existingVC.getStatistics()== null) {
+ jlog.info("Update statistics for VC: {}", vcId);
+ String statistics = vcService.computeStatisticsForVC(existingVC,
+ QueryType.VIRTUAL_CORPUS, apiVersion);
+ vcService.updateVCStatistics(existingVC,statistics);
+ }
}
catch (KustvaktException e) {
// VC doesn't exist in the DB
diff --git a/src/main/java/de/ids_mannheim/korap/service/QueryService.java b/src/main/java/de/ids_mannheim/korap/service/QueryService.java
index d2c7f6b..cd93de2 100644
--- a/src/main/java/de/ids_mannheim/korap/service/QueryService.java
+++ b/src/main/java/de/ids_mannheim/korap/service/QueryService.java
@@ -154,7 +154,7 @@
return dtos;
}
- private String computeStatisticsForVC (QueryDO query, QueryType queryType,
+ public String computeStatisticsForVC (QueryDO query, QueryType queryType,
double apiVersion)
throws KustvaktException {
if (config.isVcListStatisticsEnabled() &&
@@ -258,10 +258,12 @@
String corpusQuery = newQuery.getCorpusQuery();
String query = newQuery.getQuery();
String queryLanguage = newQuery.getQueryLanguage();
+ String statistics = null;
if (corpusQuery != null && !corpusQuery.isEmpty()) {
koralQuery = serializeCorpusQuery(corpusQuery, apiVersion);
requiredAccess = determineRequiredAccess(newQuery.isCached(),
queryName, koralQuery, apiVersion);
+ statistics = krill.getStatistics(koralQuery);
}
else if (query != null && !query.isEmpty() && queryLanguage != null
&& !queryLanguage.isEmpty()) {
@@ -291,7 +293,7 @@
queryDao.editQuery(existingQuery, queryName, type, requiredAccess,
koralQuery, newQuery.getDefinition(), newQuery.getDescription(),
newQuery.getStatus(), newQuery.isCached(), query,
- queryLanguage);
+ queryLanguage, statistics);
}
private void publishQuery (int queryId, String queryCreator,
@@ -412,17 +414,23 @@
jlog.debug("Storing query: " + queryName + "in the database ");
}
+ String statistics = null;
+ if (queryType.equals(QueryType.VIRTUAL_CORPUS)) {
+ statistics = krill.getStatistics(koralQuery);
+ }
+
int queryId = 0;
try {
if (existingQuery==null) {
queryId = queryDao.createQuery(queryName, type, queryType,
requiredAccess, koralQuery, definition, description,
- status, isCached, queryCreator, query, queryLanguage);
+ status, isCached, queryCreator, query, queryLanguage,
+ statistics);
}
else {
queryDao.editQuery(existingQuery, queryName, type,
requiredAccess, koralQuery, definition, description,
- status, isCached, query, queryLanguage);
+ status, isCached, query, queryLanguage, statistics);
}
}
@@ -443,7 +451,13 @@
publishQuery(queryId, queryCreator, queryName);
}
}
-
+
+ public void updateVCStatistics (QueryDO existingQuery, String statistics)
+ throws KustvaktException {
+ queryDao.editQuery(existingQuery, null, null, null, null, null, null,
+ null, existingQuery.isCached(), null, null, statistics);
+ }
+
public String serializeCorpusQuery (String corpusQuery, double apiVersion)
throws KustvaktException {
QuerySerializer serializer = new QuerySerializer(apiVersion);
@@ -568,7 +582,7 @@
}
queryDao.editQuery(query, null, queryType, null, null,
- null, null, null, query.isCached(), null, null);
+ null, null, null, query.isCached(), null, null, null);
}
}
diff --git a/src/main/resources/db/sqlite/V4.4__query_statistics.sql b/src/main/resources/db/sqlite/V4.4__query_statistics.sql
new file mode 100644
index 0000000..ced83c4
--- /dev/null
+++ b/src/main/resources/db/sqlite/V4.4__query_statistics.sql
@@ -0,0 +1,2 @@
+ALTER TABLE query
+ADD COLUMN statistics TEXT DEFAULT NULL;
\ No newline at end of file
diff --git a/src/test/java/de/ids_mannheim/korap/cache/NamedVCLoaderTest.java b/src/test/java/de/ids_mannheim/korap/cache/NamedVCLoaderTest.java
index 1b3eead..037e0a2 100644
--- a/src/test/java/de/ids_mannheim/korap/cache/NamedVCLoaderTest.java
+++ b/src/test/java/de/ids_mannheim/korap/cache/NamedVCLoaderTest.java
@@ -52,6 +52,19 @@
}
@Test
+ public void testVCName2 ()
+ throws IOException, QueryException, KustvaktException {
+ String vcId = "c";
+ vcLoader.loadVCToCache(vcId, "/vc/corp-w-gesamt.2017-i.17.10.17.jsonld");
+ assertTrue(VirtualCorpusCache.contains(vcId));
+ Map<String, DocBits> cachedData = VirtualCorpusCache.retrieve(vcId);
+ assertTrue(cachedData.size() > 0);
+ QueryDO vc = dao.retrieveQueryByName(vcId, "system");
+ assertNotNull(vc);
+ clean(vcId, vc);
+ }
+
+ @Test
public void testNamedVCLoader ()
throws IOException, QueryException, KustvaktException {
String vcId = "named-vc1";
diff --git a/src/test/java/de/ids_mannheim/korap/dao/VirtualCorpusDaoTest.java b/src/test/java/de/ids_mannheim/korap/dao/VirtualCorpusDaoTest.java
index d5773a9..7311947 100644
--- a/src/test/java/de/ids_mannheim/korap/dao/VirtualCorpusDaoTest.java
+++ b/src/test/java/de/ids_mannheim/korap/dao/VirtualCorpusDaoTest.java
@@ -45,7 +45,7 @@
int id = dao.createQuery("system-vc-2", ResourceType.SYSTEM,
QueryType.VIRTUAL_CORPUS, User.CorpusAccess.FREE,
"corpusSigle=GOE", "definition", "description", "experimental",
- false, "test class", null, null);
+ false, "test class", null, null, null);
// select vc
List<QueryDO> vcList = dao.retrieveQueryByType(ResourceType.SYSTEM,
null, QueryType.VIRTUAL_CORPUS);
@@ -70,7 +70,7 @@
dao.createQuery("system-vc", ResourceType.SYSTEM,
QueryType.VIRTUAL_CORPUS, User.CorpusAccess.FREE,
"corpusSigle=GOE", "definition", "description",
- "experimental", false, "system", null, null);
+ "experimental", false, "system", null, null, null);
});
String msg = exception.getMessage();
diff --git a/src/test/java/de/ids_mannheim/korap/web/controller/oauth2/OAuth2AdminControllerTest.java b/src/test/java/de/ids_mannheim/korap/web/controller/oauth2/OAuth2AdminControllerTest.java
index 80fe019..d0c5d5f 100644
--- a/src/test/java/de/ids_mannheim/korap/web/controller/oauth2/OAuth2AdminControllerTest.java
+++ b/src/test/java/de/ids_mannheim/korap/web/controller/oauth2/OAuth2AdminControllerTest.java
@@ -12,6 +12,8 @@
import org.apache.http.entity.ContentType;
import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+import org.junit.jupiter.api.MethodOrderer;
import org.springframework.beans.factory.annotation.Autowired;
import com.fasterxml.jackson.databind.JsonNode;
@@ -25,6 +27,7 @@
import de.ids_mannheim.korap.oauth2.dao.RefreshTokenDao;
import de.ids_mannheim.korap.utils.JsonUtils;
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@Order(Integer.MAX_VALUE) // make sure this runs as last test as it removes tokens
public class OAuth2AdminControllerTest extends OAuth2TestBase {
@@ -185,4 +188,4 @@
assertEquals(node.at("/errors/0/1").asText(),
"Access token is invalid");
}
-}
+}
\ No newline at end of file