Added an API retrieving fields of a virtual corpus.
Change-Id: I577e29bae1b3112ccb16a2d6b2c65d06695198c0
diff --git a/full/Changes b/full/Changes
index d84e29a..c32a2f7 100644
--- a/full/Changes
+++ b/full/Changes
@@ -20,6 +20,8 @@
- Updated tests.
2022-01-25
- Added show-tokens option to the search API.
+2022-01-31
+ - Added an API retrieving fields of a virtual corpus.
# version 0.64.1
diff --git a/full/src/main/java/de/ids_mannheim/korap/authentication/KustvaktAuthenticationManager.java b/full/src/main/java/de/ids_mannheim/korap/authentication/KustvaktAuthenticationManager.java
index 3fd5722..c135506 100644
--- a/full/src/main/java/de/ids_mannheim/korap/authentication/KustvaktAuthenticationManager.java
+++ b/full/src/main/java/de/ids_mannheim/korap/authentication/KustvaktAuthenticationManager.java
@@ -315,6 +315,7 @@
return false;
}
+ @Deprecated
private User authenticateShib(Map<String, Object> attributes) throws KustvaktException {
// todo use persistent id, since eppn is not unique
String eppn = (String) attributes.get(Attributes.EPPN);
@@ -648,8 +649,8 @@
entHandler.createAccount(user);
- s.setUserId(user.getId());
- d.setUserId(user.getId());
+// s.setUserId(user.getId());
+// d.setUserId(user.getId());
UserDataDbIface dao = BeansFactory.getTypeFactory().getTypeInterfaceBean(userdatadaos, UserDetails.class);
assert dao != null;
diff --git a/full/src/main/java/de/ids_mannheim/korap/service/QueryService.java b/full/src/main/java/de/ids_mannheim/korap/service/QueryService.java
index 6cfe8ee..7e042f1 100644
--- a/full/src/main/java/de/ids_mannheim/korap/service/QueryService.java
+++ b/full/src/main/java/de/ids_mannheim/korap/service/QueryService.java
@@ -65,8 +65,7 @@
@Service
public class QueryService {
- public static Logger jlog =
- LogManager.getLogger(QueryService.class);
+ public static Logger jlog = LogManager.getLogger(QueryService.class);
public static boolean DEBUG = false;
@@ -99,13 +98,13 @@
}
}
- public List<QueryDto> listOwnerQuery (String username,
- String queryCreator, QueryType queryType) throws KustvaktException {
+ public List<QueryDto> listOwnerQuery (String username, String queryCreator,
+ QueryType queryType) throws KustvaktException {
verifyUsername(username, queryCreator);
List<QueryDO> list = queryDao.retrieveOwnerQuery(username, queryType);
return createQueryDtos(list, queryType);
}
-
+
public List<QueryDto> listSystemQuery (QueryType queryType)
throws KustvaktException {
List<QueryDO> list = queryDao.retrieveQueryByType(ResourceType.SYSTEM,
@@ -130,14 +129,12 @@
else {
username = authenticatedUsername;
}
- List<QueryDO> list =
- queryDao.retrieveQueryByUser(username, queryType);
+ List<QueryDO> list = queryDao.retrieveQueryByUser(username, queryType);
return createQueryDtos(list, queryType);
}
- public List<QueryDto> listQueryByType (String username,
- String createdBy, ResourceType type, QueryType queryType)
- throws KustvaktException {
+ public List<QueryDto> listQueryByType (String username, String createdBy,
+ ResourceType type, QueryType queryType) throws KustvaktException {
boolean isAdmin = adminDao.isAdmin(username);
@@ -153,21 +150,19 @@
}
}
- private ArrayList<QueryDto> createQueryDtos (
- List<QueryDO> queryList, QueryType queryType)
- throws KustvaktException {
+ private ArrayList<QueryDto> createQueryDtos (List<QueryDO> queryList,
+ QueryType queryType) throws KustvaktException {
ArrayList<QueryDto> dtos = new ArrayList<>(queryList.size());
QueryDO query;
Iterator<QueryDO> i = queryList.iterator();
while (i.hasNext()) {
query = i.next();
-// String json = query.getKoralQuery();
+ // String json = query.getKoralQuery();
String statistics = null;
-// if (queryType.equals(QueryType.VIRTUAL_CORPUS)) {
-// statistics = krill.getStatistics(json);
-// }
- QueryDto dto =
- converter.createQueryDto(query, statistics);
+ // if (queryType.equals(QueryType.VIRTUAL_CORPUS)) {
+ // statistics = krill.getStatistics(json);
+ // }
+ QueryDto dto = converter.createQueryDto(query, statistics);
dtos.add(dto);
}
return dtos;
@@ -181,8 +176,7 @@
if (query == null) {
String code = createdBy + "/" + queryName;
throw new KustvaktException(StatusCodes.NO_RESOURCE_FOUND,
- "Query " + code + " is not found.",
- String.valueOf(code));
+ "Query " + code + " is not found.", String.valueOf(code));
}
else if (query.getCreatedBy().equals(username)
|| adminDao.isAdmin(username)) {
@@ -196,7 +190,7 @@
}
if (type.equals(QueryType.VIRTUAL_CORPUS)
&& VirtualCorpusCache.contains(queryName)) {
- VirtualCorpusCache.delete(queryName);
+ VirtualCorpusCache.delete(queryName);
}
queryDao.deleteQuery(query);
}
@@ -211,7 +205,7 @@
verifyUsername(username, queryCreator);
QueryDO query = queryDao.retrieveQueryByName(queryName, queryCreator);
-
+
if (query == null) {
storeQuery(queryJson, queryName, queryCreator, username);
return Status.CREATED;
@@ -238,8 +232,8 @@
String queryLanguage = newQuery.getQueryLanguage();
if (corpusQuery != null && !corpusQuery.isEmpty()) {
koralQuery = serializeCorpusQuery(corpusQuery);
- requiredAccess = determineRequiredAccess(newQuery.isCached(), queryName,
- koralQuery);
+ requiredAccess = determineRequiredAccess(newQuery.isCached(),
+ queryName, koralQuery);
}
else if (query != null && !query.isEmpty() && queryLanguage != null
&& !queryLanguage.isEmpty()) {
@@ -251,8 +245,8 @@
if (existingQuery.getType().equals(ResourceType.PUBLISHED)) {
// withdraw from publication
if (!type.equals(ResourceType.PUBLISHED)) {
- QueryAccess hiddenAccess =
- accessDao.retrieveHiddenAccess(existingQuery.getId());
+ QueryAccess hiddenAccess = accessDao
+ .retrieveHiddenAccess(existingQuery.getId());
deleteQueryAccess(hiddenAccess.getId(), "system");
int groupId = hiddenAccess.getUserGroup().getId();
userGroupService.deleteAutoHiddenGroup(groupId, "system");
@@ -269,7 +263,8 @@
queryDao.editQuery(existingQuery, queryName, type, requiredAccess,
koralQuery, newQuery.getDefinition(), newQuery.getDescription(),
- newQuery.getStatus(), newQuery.isCached(), query, queryLanguage);
+ newQuery.getStatus(), newQuery.isCached(), query,
+ queryLanguage);
}
private void publishQuery (int queryId) throws KustvaktException {
@@ -291,7 +286,7 @@
+ ". Hidden access exists! Access id: " + access.getId());
}
}
-
+
public void storeQuery (QueryJson query, String queryName,
String queryCreator, String username) throws KustvaktException {
String koralQuery = null;
@@ -323,22 +318,22 @@
ParameterChecker.checkObjectValue(type, "type");
if (!queryNamePattern.matcher(queryName).matches()) {
- throw new KustvaktException(StatusCodes.INVALID_ARGUMENT,
- queryType.displayName() + " name must only contain "
- + "letters, numbers, underscores, hypens and spaces",
+ throw new KustvaktException(StatusCodes.INVALID_ARGUMENT, queryType
+ .displayName() + " name must only contain "
+ + "letters, numbers, underscores, hypens and spaces",
queryName);
}
- if (type.equals(ResourceType.SYSTEM)){
+ if (type.equals(ResourceType.SYSTEM)) {
if (adminDao.isAdmin(username)) {
- queryCreator="system";
+ queryCreator = "system";
}
else if (!username.equals("system")) {
throw new KustvaktException(StatusCodes.AUTHORIZATION_FAILED,
- "Unauthorized operation for user: " + username, username);
+ "Unauthorized operation for user: " + username,
+ username);
}
}
-
CorpusAccess requiredAccess = CorpusAccess.PUB;
if (queryType.equals(QueryType.VIRTUAL_CORPUS)) {
@@ -346,10 +341,10 @@
determineRequiredAccess(isCached, queryName, koralQuery);
}
- if (DEBUG){
+ if (DEBUG) {
jlog.debug("Storing query: " + queryName + "in the database ");
}
-
+
int queryId = 0;
try {
queryId = queryDao.createQuery(queryName, type, queryType,
@@ -453,8 +448,7 @@
if (query == null) {
String code = createdBy + "/" + queryName;
throw new KustvaktException(StatusCodes.NO_RESOURCE_FOUND,
- "Query " + code + " is not found.",
- String.valueOf(code));
+ "Query " + code + " is not found.", String.valueOf(code));
}
if (!username.equals(query.getCreatedBy())
&& !adminDao.isAdmin(username)) {
@@ -546,10 +540,12 @@
List<QueryAccess> accessList;
if (adminDao.isAdmin(username)) {
- accessList = accessDao.retrieveAllAccessByQuery(queryCreator, queryName);
+ accessList =
+ accessDao.retrieveAllAccessByQuery(queryCreator, queryName);
}
else {
- accessList = accessDao.retrieveActiveAccessByQuery(queryCreator, queryName);
+ accessList = accessDao.retrieveActiveAccessByQuery(queryCreator,
+ queryName);
List<QueryAccess> filteredAccessList = new ArrayList<>();
for (QueryAccess access : accessList) {
UserGroup userGroup = access.getUserGroup();
@@ -617,22 +613,38 @@
}
}
-
+
public JsonNode retrieveKoralQuery (String username, String queryName,
String createdBy, QueryType queryType) throws KustvaktException {
- QueryDO query = searchQueryByName(username, queryName, createdBy, queryType);
+ QueryDO query =
+ searchQueryByName(username, queryName, createdBy, queryType);
String koralQuery = query.getKoralQuery();
- JsonNode kq = JsonUtils.readTree(koralQuery);
+ JsonNode kq = JsonUtils.readTree(koralQuery);
return kq;
}
+ public JsonNode retrieveFieldValues (String username, String queryName,
+ String createdBy, QueryType queryType, String fieldName)
+ throws KustvaktException {
+ if (fieldName.equals("tokens") || fieldName.equals("base")) {
+ throw new KustvaktException(StatusCodes.NOT_ALLOWED,
+ "Retrieving values of field "+fieldName+" is not allowed.");
+ }
+ else {
+ QueryDO query = searchQueryByName(username, queryName, createdBy,
+ queryType);
+ String koralQuery = query.getKoralQuery();
+ return krill.getFieldValuesForVC(koralQuery, fieldName);
+ }
+ }
+
public QueryDO searchQueryByName (String username, String queryName,
String createdBy, QueryType queryType) throws KustvaktException {
QueryDO query = queryDao.retrieveQueryByName(queryName, createdBy);
if (query == null) {
String code = createdBy + "/" + queryName;
throw new KustvaktException(StatusCodes.NO_RESOURCE_FOUND,
- queryType.displayName()+ " " + code + " is not found.",
+ queryType.displayName() + " " + code + " is not found.",
String.valueOf(code));
}
checkQueryAccess(query, username);
@@ -641,16 +653,19 @@
public QueryDto retrieveQueryByName (String username, String queryName,
String createdBy, QueryType queryType) throws KustvaktException {
- QueryDO query = searchQueryByName(username, queryName, createdBy, queryType);
-// String json = query.getKoralQuery();
+ QueryDO query =
+ searchQueryByName(username, queryName, createdBy, queryType);
+ // String json = query.getKoralQuery();
String statistics = null;
-// long start,end;
-// start = System.currentTimeMillis();
-// if (query.getQueryType().equals(QueryType.VIRTUAL_CORPUS)) {
-// statistics = krill.getStatistics(json);
-// }
-// end = System.currentTimeMillis();
-// jlog.debug("{} statistics duration: {}", queryName, (end - start));
+ // long start,end;
+ // start = System.currentTimeMillis();
+ // if (query.getQueryType().equals(QueryType.VIRTUAL_CORPUS))
+ // {
+ // statistics = krill.getStatistics(json);
+ // }
+ // end = System.currentTimeMillis();
+ // jlog.debug("{} statistics duration: {}", queryName, (end -
+ // start));
return converter.createQueryDto(query, statistics);
}
@@ -659,8 +674,8 @@
QueryDO query = queryDao.retrieveQueryById(queryId);
checkQueryAccess(query, username);
-// String json = query.getKoralQuery();
-// String statistics = krill.getStatistics(json);
+ // String json = query.getKoralQuery();
+ // String statistics = krill.getStatistics(json);
return converter.createQueryDto(query, null);
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/web/controller/VirtualCorpusController.java b/full/src/main/java/de/ids_mannheim/korap/web/controller/VirtualCorpusController.java
index 8a9f16f..31a2c19 100644
--- a/full/src/main/java/de/ids_mannheim/korap/web/controller/VirtualCorpusController.java
+++ b/full/src/main/java/de/ids_mannheim/korap/web/controller/VirtualCorpusController.java
@@ -173,6 +173,25 @@
}
}
+ @GET
+ @Path("/field/~{createdBy}/{vcName}")
+ @Produces(MediaType.APPLICATION_JSON + ";charset=utf-8")
+ public JsonNode retrieveVCField (
+ @Context SecurityContext securityContext,
+ @PathParam("createdBy") String createdBy,
+ @PathParam("vcName") String vcName,
+ @QueryParam("fieldName") String fieldName) {
+ TokenContext context =
+ (TokenContext) securityContext.getUserPrincipal();
+ try {
+ scopeService.verifyScope(context, OAuth2Scope.VC_INFO);
+ return service.retrieveFieldValues(context.getUsername(), vcName,
+ createdBy, QueryType.VIRTUAL_CORPUS, fieldName);
+ }
+ catch (KustvaktException e) {
+ throw kustvaktResponseHandler.throwit(e);
+ }
+ }
/**
* Lists all virtual corpora available to the authenticated user.
diff --git a/full/src/test/java/de/ids_mannheim/korap/web/controller/VirtualCorpusFieldTest.java b/full/src/test/java/de/ids_mannheim/korap/web/controller/VirtualCorpusFieldTest.java
new file mode 100644
index 0000000..583629b
--- /dev/null
+++ b/full/src/test/java/de/ids_mannheim/korap/web/controller/VirtualCorpusFieldTest.java
@@ -0,0 +1,135 @@
+package de.ids_mannheim.korap.web.controller;
+
+import static org.junit.Assert.assertEquals;
+
+import java.io.IOException;
+
+import org.apache.http.entity.ContentType;
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.net.HttpHeaders;
+import com.sun.jersey.api.client.ClientHandlerException;
+import com.sun.jersey.api.client.ClientResponse;
+import com.sun.jersey.api.client.ClientResponse.Status;
+import com.sun.jersey.api.client.UniformInterfaceException;
+
+import de.ids_mannheim.korap.authentication.http.HttpAuthorizationHandler;
+import de.ids_mannheim.korap.cache.VirtualCorpusCache;
+import de.ids_mannheim.korap.config.Attributes;
+import de.ids_mannheim.korap.config.NamedVCLoader;
+import de.ids_mannheim.korap.dao.QueryDao;
+import de.ids_mannheim.korap.entity.QueryDO;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.exceptions.StatusCodes;
+import de.ids_mannheim.korap.util.QueryException;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+public class VirtualCorpusFieldTest extends VirtualCorpusTestBase {
+
+ @Autowired
+ private NamedVCLoader vcLoader;
+ @Autowired
+ private QueryDao dao;
+
+ private JsonNode testRetrieveField (String username, String vcName,
+ String field) throws UniformInterfaceException,
+ ClientHandlerException, KustvaktException {
+ ClientResponse response = resource().path(API_VERSION).path("vc")
+ .path("field").path("~" + username).path(vcName)
+ .queryParam("fieldName", field)
+ .header(Attributes.AUTHORIZATION, HttpAuthorizationHandler
+ .createBasicAuthorizationHeaderValue("dory", "pass"))
+ .header(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+
+ assertEquals(Status.OK.getStatusCode(), response.getStatus());
+ String entity = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+ return node;
+ }
+
+ private void testRetrieveProhibitedField (String username, String vcName,
+ String field) throws UniformInterfaceException,
+ ClientHandlerException, KustvaktException {
+ ClientResponse response = resource().path(API_VERSION).path("vc")
+ .path("field").path("~" + username).path(vcName)
+ .queryParam("fieldName", field)
+ .header(Attributes.AUTHORIZATION, HttpAuthorizationHandler
+ .createBasicAuthorizationHeaderValue("dory", "pass"))
+ .header(HttpHeaders.CONTENT_TYPE, ContentType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+
+ assertEquals(Status.BAD_REQUEST.getStatusCode(), response.getStatus());
+ String entity = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(entity);
+ assertEquals(StatusCodes.NOT_ALLOWED, node.at("/errors/0/0").asInt());
+ }
+
+ private void deleteVcFromDB (String vcName) throws KustvaktException {
+ QueryDO vc = dao.retrieveQueryByName(vcName, "system");
+ dao.deleteQuery(vc);
+ vc = dao.retrieveQueryByName(vcName, "system");
+ assertEquals(null, vc);
+ }
+
+ @Test
+ public void testRetrieveFieldsNamedVC1 ()
+ throws IOException, QueryException, KustvaktException {
+
+ vcLoader.loadVCToCache("named-vc1", "/vc/named-vc1.jsonld");
+
+ JsonNode n = testRetrieveField("system", "named-vc1", "textSigle");
+ assertEquals(
+ "http://korap.ids-mannheim.de/ns/KoralQuery/v0.3/context.jsonld",
+ n.at("/@context").asText());
+ assertEquals("textSigle", n.at("/corpus/key").asText());
+ assertEquals(2, n.at("/corpus/value").size());
+
+ n = testRetrieveField("system", "named-vc1", "author");
+ assertEquals(2, n.at("/corpus/value").size());
+ assertEquals("Goethe, Johann Wolfgang von",
+ n.at("/corpus/value/0").asText());
+
+ testRetrieveUnknownTokens();
+ testRetrieveProhibitedField("system", "named-vc1", "tokens");
+ testRetrieveProhibitedField("system", "named-vc1", "base");
+
+ VirtualCorpusCache.delete("named-vc1");
+
+ deleteVcFromDB("named-vc1");
+ }
+
+ private void testRetrieveUnknownTokens () throws UniformInterfaceException,
+ ClientHandlerException, KustvaktException {
+ JsonNode n = testRetrieveField("system", "named-vc1", "unknown");
+ assertEquals("unknown", n.at("/corpus/key").asText());
+ assertEquals(0, n.at("/corpus/value").size());
+ }
+
+ @Test
+ public void testRetrieveTextSigleNamedVC2 ()
+ throws IOException, QueryException, KustvaktException {
+ vcLoader.loadVCToCache("named-vc2", "/vc/named-vc2.jsonld");
+
+ JsonNode n = testRetrieveField("system", "named-vc2", "textSigle");
+ assertEquals(2, n.at("/corpus/value").size());
+ VirtualCorpusCache.delete("named-vc2");
+ deleteVcFromDB("named-vc2");
+ }
+
+ @Test
+ public void testRetrieveTextSigleNamedVC3 ()
+ throws IOException, QueryException, KustvaktException {
+ vcLoader.loadVCToCache("named-vc3", "/vc/named-vc3.jsonld");
+
+ JsonNode n = testRetrieveField("system", "named-vc3", "textSigle");
+ n = n.at("/corpus/value");
+ assertEquals(1, n.size());
+ assertEquals("GOE/AGI/00000", n.get(0).asText());
+
+ VirtualCorpusCache.delete("named-vc3");
+ deleteVcFromDB("named-vc3");
+ }
+}
diff --git a/full/src/test/resources/vc/named-vc3.jsonld b/full/src/test/resources/vc/named-vc3.jsonld
new file mode 100644
index 0000000..a690744
--- /dev/null
+++ b/full/src/test/resources/vc/named-vc3.jsonld
@@ -0,0 +1,9 @@
+{
+ "collection": {
+ "@type": "koral:doc",
+ "key": "title",
+ "match": "match:eq",
+ "type": "type:string",
+ "value": "Italienische Reise"
+ }
+}