Implemented VirtualCorpusRewrite.
Change-Id: I937cfa78db9be0602b17222f37de72cf3b63027c
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java b/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
index 57d78b2..85360c7 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
@@ -5,6 +5,7 @@
import org.springframework.beans.factory.annotation.Autowired;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
import de.ids_mannheim.korap.oauth2.constant.OAuth2Scope;
import de.ids_mannheim.korap.oauth2.dao.AccessScopeDao;
import de.ids_mannheim.korap.util.QueryException;
@@ -25,7 +26,7 @@
public Initializator () {}
- public void init () throws IOException, QueryException {
+ public void init () throws IOException, QueryException, KustvaktException {
setInitialAccessScope();
loader.loadVCToCache();
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java b/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
index ff303b1..4bee9be 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
@@ -13,6 +13,12 @@
import org.springframework.stereotype.Component;
import de.ids_mannheim.korap.KrillCollection;
+import de.ids_mannheim.korap.constant.VirtualCorpusType;
+import de.ids_mannheim.korap.dao.VirtualCorpusDao;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.exceptions.StatusCodes;
+import de.ids_mannheim.korap.service.VirtualCorpusService;
+import de.ids_mannheim.korap.user.User.CorpusAccess;
import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.web.SearchKrill;
@@ -22,10 +28,14 @@
private FullConfiguration config;
@Autowired
private SearchKrill searchKrill;
-
+ @Autowired
+ private VirtualCorpusDao vcDao;
+ @Autowired
+ private VirtualCorpusService vcService;
+
private static Logger jlog = LogManager.getLogger(NamedVCLoader.class);
- public void loadVCToCache () throws IOException, QueryException {
+ public void loadVCToCache () throws IOException, QueryException, KustvaktException {
String dir = config.getNamedVCPath();
File d = new File(dir);
@@ -66,18 +76,39 @@
continue;
}
jlog.debug(
- "READ " + file.getName() + " duration: " + (end - start));
+ "READ " + filename + " duration: " + (end - start));
- KrillCollection collection = new KrillCollection(json);
- collection.setIndex(searchKrill.getIndex());
-
- if (collection != null) {
- collection.storeInCache(filename);
- }
- end = System.currentTimeMillis();
- jlog.debug(filename + "caching duration: " + (end - start));
- jlog.debug("memory cache: "
- + KrillCollection.cache.calculateInMemorySize());
+ cacheVC(json, filename);
+ storeVC(filename, json);
}
}
+
+ private void cacheVC (String json, String filename) throws IOException, QueryException {
+ long start, end;
+ start = System.currentTimeMillis();
+
+ KrillCollection collection = new KrillCollection(json);
+ collection.setIndex(searchKrill.getIndex());
+
+ if (collection != null) {
+ collection.storeInCache(filename);
+ }
+ end = System.currentTimeMillis();
+ jlog.info(filename + " caching duration: " + (end - start));
+ jlog.debug("memory cache: "
+ + KrillCollection.cache.calculateInMemorySize());
+ }
+
+ private void storeVC (String name, String koralQuery) throws KustvaktException {
+ if (!VirtualCorpusService.wordPattern.matcher(name).matches()) {
+ throw new KustvaktException(StatusCodes.INVALID_ARGUMENT,
+ "Virtual corpus name must only contains letters, numbers, "
+ + "underscores, hypens and spaces",
+ name);
+ }
+ CorpusAccess requiredAccess = vcService.determineRequiredAccess(koralQuery);
+ vcDao.createVirtualCorpus(name, VirtualCorpusType.SYSTEM,
+ requiredAccess, koralQuery, null,
+ null, null, true, "system");
+ }
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/dao/VirtualCorpusDao.java b/full/src/main/java/de/ids_mannheim/korap/dao/VirtualCorpusDao.java
index 9e9983c..c8006aa 100644
--- a/full/src/main/java/de/ids_mannheim/korap/dao/VirtualCorpusDao.java
+++ b/full/src/main/java/de/ids_mannheim/korap/dao/VirtualCorpusDao.java
@@ -36,9 +36,10 @@
import de.ids_mannheim.korap.user.User.CorpusAccess;
import de.ids_mannheim.korap.utils.ParameterChecker;
-/** VirtualCorpusDao manages SQL queries regarding virtual corpora,
- * e.g. retrieving and storing virtual corpora.
- *
+/**
+ * VirtualCorpusDao manages SQL queries regarding virtual corpora,
+ * e.g. retrieving and storing virtual corpora.
+ *
* @author margaretha
*
*/
@@ -50,19 +51,20 @@
private EntityManager entityManager;
public int createVirtualCorpus (String name, VirtualCorpusType type,
- CorpusAccess requiredAccess, String corpusQuery, String definition,
- String description, String status, String createdBy)
- throws KustvaktException {
+ CorpusAccess requiredAccess, String koralQuery, String definition,
+ String description, String status, boolean isCached,
+ String createdBy) throws KustvaktException {
VirtualCorpus vc = new VirtualCorpus();
vc.setName(name);
vc.setType(type);
vc.setRequiredAccess(requiredAccess);
- vc.setCorpusQuery(corpusQuery);
+ vc.setKoralQuery(koralQuery);
vc.setDefinition(definition);
vc.setDescription(description);
vc.setStatus(status);
vc.setCreatedBy(createdBy);
+ vc.setCached(isCached);
entityManager.persist(vc);
return vc.getId();
@@ -70,7 +72,7 @@
public void editVirtualCorpus (VirtualCorpus vc, String name,
VirtualCorpusType type, CorpusAccess requiredAccess,
- String corpusQuery, String definition, String description,
+ String koralQuery, String definition, String description,
String status) throws KustvaktException {
if (name != null && !name.isEmpty()) {
@@ -82,8 +84,8 @@
if (requiredAccess != null) {
vc.setRequiredAccess(requiredAccess);
}
- if (corpusQuery != null) {
- vc.setCorpusQuery(corpusQuery);
+ if (koralQuery != null) {
+ vc.setKoralQuery(koralQuery);
}
if (definition != null && !definition.isEmpty()) {
vc.setDefinition(definition);
@@ -97,22 +99,28 @@
entityManager.merge(vc);
}
- public void deleteVirtualCorpus (VirtualCorpus vc) throws KustvaktException {
- if (!entityManager.contains(vc)){
+ public void deleteVirtualCorpus (VirtualCorpus vc)
+ throws KustvaktException {
+ if (!entityManager.contains(vc)) {
vc = entityManager.merge(vc);
}
entityManager.remove(vc);
-
+
}
- /** System admin function.
+ /**
+ * System admin function.
*
- * Retrieves virtual corpus by creator and type. If type is not specified,
- * retrieves virtual corpora of all types. If createdBy is not specified,
- * retrieves virtual corpora of all users.
+ * Retrieves virtual corpus by creator and type. If type is not
+ * specified,
+ * retrieves virtual corpora of all types. If createdBy is not
+ * specified,
+ * retrieves virtual corpora of all users.
*
- * @param type {@link VirtualCorpusType}
- * @param createdBy username of the virtual corpus creator
+ * @param type
+ * {@link VirtualCorpusType}
+ * @param createdBy
+ * username of the virtual corpus creator
* @return a list of {@link VirtualCorpus}
* @throws KustvaktException
*/
@@ -172,6 +180,40 @@
return vc;
}
+ public VirtualCorpus retrieveVCByName (String vcName, String createdBy)
+ throws KustvaktException {
+ ParameterChecker.checkStringValue(createdBy, "createdBy");
+ ParameterChecker.checkStringValue(vcName, "vcName");
+
+ CriteriaBuilder builder = entityManager.getCriteriaBuilder();
+ CriteriaQuery<VirtualCorpus> query =
+ builder.createQuery(VirtualCorpus.class);
+
+ Root<VirtualCorpus> virtualCorpus = query.from(VirtualCorpus.class);
+
+ Predicate condition = builder.and(
+ builder.equal(virtualCorpus.get(VirtualCorpus_.createdBy),
+ createdBy),
+ builder.equal(virtualCorpus.get(VirtualCorpus_.name), vcName));
+
+ query.select(virtualCorpus);
+ query.where(condition);
+
+ Query q = entityManager.createQuery(query);
+ VirtualCorpus vc = null;
+ try {
+ vc = (VirtualCorpus) q.getSingleResult();
+ }
+ catch (NoResultException e) {
+ String vcCode = createdBy + "/" + vcName;
+ throw new KustvaktException(StatusCodes.NO_RESULT_FOUND,
+ "No result found for query: retrieve virtual corpus by name "
+ + vcCode,
+ String.valueOf(vcCode), e);
+ }
+ return vc;
+ }
+
@SuppressWarnings("unchecked")
public List<VirtualCorpus> retrieveOwnerVC (String userId)
throws KustvaktException {
@@ -225,11 +267,11 @@
Join<VirtualCorpus, VirtualCorpusAccess> access =
virtualCorpus.join(VirtualCorpus_.virtualCorpusAccess);
- // Predicate corpusStatus = builder.and(
- // builder.notEqual(access.get(VirtualCorpusAccess_.status),
- // VirtualCorpusAccessStatus.HIDDEN),
- // builder.notEqual(access.get(VirtualCorpusAccess_.status),
- // VirtualCorpusAccessStatus.DELETED));
+ // Predicate corpusStatus = builder.and(
+ // builder.notEqual(access.get(VirtualCorpusAccess_.status),
+ // VirtualCorpusAccessStatus.HIDDEN),
+ // builder.notEqual(access.get(VirtualCorpusAccess_.status),
+ // VirtualCorpusAccessStatus.DELETED));
Predicate corpusStatus =
builder.notEqual(access.get(VirtualCorpusAccess_.status),
@@ -270,7 +312,6 @@
builder.equal(virtualCorpus.get(VirtualCorpus_.type),
VirtualCorpusType.SYSTEM));
-
query.select(virtualCorpus);
query.where(predicate);
query.distinct(true);
diff --git a/full/src/main/java/de/ids_mannheim/korap/dto/converter/VirtualCorpusConverter.java b/full/src/main/java/de/ids_mannheim/korap/dto/converter/VirtualCorpusConverter.java
index 52cbfe6..9fec23d 100644
--- a/full/src/main/java/de/ids_mannheim/korap/dto/converter/VirtualCorpusConverter.java
+++ b/full/src/main/java/de/ids_mannheim/korap/dto/converter/VirtualCorpusConverter.java
@@ -31,7 +31,7 @@
dto.setStatus(vc.getStatus());
dto.setDescription(vc.getDescription());
dto.setType(vc.getType().displayName());
- dto.setKoralQuery(vc.getCorpusQuery());
+ dto.setKoralQuery(vc.getKoralQuery());
JsonNode node = JsonUtils.readTree(statistics);
int numberOfDoc = node.at("/documents").asInt();
diff --git a/full/src/main/java/de/ids_mannheim/korap/entity/VirtualCorpus.java b/full/src/main/java/de/ids_mannheim/korap/entity/VirtualCorpus.java
index 78f9fff..4a6aeeb 100644
--- a/full/src/main/java/de/ids_mannheim/korap/entity/VirtualCorpus.java
+++ b/full/src/main/java/de/ids_mannheim/korap/entity/VirtualCorpus.java
@@ -48,10 +48,12 @@
@Column(name = "required_access")
private CorpusAccess requiredAccess;
@Column(name = "corpus_query")
- private String corpusQuery;
+ private String koralQuery;
private String definition;
@Column(name = "created_by")
private String createdBy;
+ @Column(name = "is_cached")
+ private boolean isCached;
@OneToMany(mappedBy = "virtualCorpus", fetch = FetchType.LAZY,
cascade = CascadeType.REMOVE)
@@ -61,7 +63,7 @@
public String toString () {
return "id=" + id + ", name= " + name + ", type= " + type + ", status= "
+ status + ", description=" + description + ", requiredAccess="
- + requiredAccess + ", corpusQuery= " + corpusQuery
+ + requiredAccess + ", corpusQuery= " + koralQuery
+ ", definition= " + definition + ", createdBy= " + createdBy;
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/rewrite/CollectionRewrite.java b/full/src/main/java/de/ids_mannheim/korap/rewrite/CollectionRewrite.java
index 08c252f..091c1a8 100644
--- a/full/src/main/java/de/ids_mannheim/korap/rewrite/CollectionRewrite.java
+++ b/full/src/main/java/de/ids_mannheim/korap/rewrite/CollectionRewrite.java
@@ -144,7 +144,7 @@
KoralCollectionQueryBuilder builder = new KoralCollectionQueryBuilder();
RewriteIdentifier identifier = new KoralNode.RewriteIdentifier(
Attributes.AVAILABILITY, user.getCorpusAccess());
- JsonNode rewrittesNode;
+ JsonNode rewrittenNode;
if (jsonNode.has("collection")) {
List<String> avalabilityCopy =
@@ -159,16 +159,16 @@
builder.with(buildAvailability(avalabilityCopy));
jlog.debug("corpus query: " + builder.toString());
builder.setBaseQuery(builder.toJSON());
- rewrittesNode = builder.mergeWith(jsonNode).at("/collection");
- node.set("collection", rewrittesNode, identifier);
+ rewrittenNode = builder.mergeWith(jsonNode).at("/collection");
+ node.set("collection", rewrittenNode, identifier);
}
}
else {
builder.with(buildAvailability(userAvailabilities));
jlog.debug("corpus query: " + builder.toString());
- rewrittesNode =
+ rewrittenNode =
JsonUtils.readTree(builder.toJSON()).at("/collection");
- node.set("collection", rewrittesNode, identifier);
+ node.set("collection", rewrittenNode, identifier);
}
jlog.debug("REWRITES: " + node.at("/collection").toString());
diff --git a/full/src/main/java/de/ids_mannheim/korap/rewrite/FullRewriteHandler.java b/full/src/main/java/de/ids_mannheim/korap/rewrite/FullRewriteHandler.java
index 0926a09..21bcc41 100644
--- a/full/src/main/java/de/ids_mannheim/korap/rewrite/FullRewriteHandler.java
+++ b/full/src/main/java/de/ids_mannheim/korap/rewrite/FullRewriteHandler.java
@@ -1,26 +1,34 @@
package de.ids_mannheim.korap.rewrite;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
import de.ids_mannheim.korap.config.FullConfiguration;
import de.ids_mannheim.korap.resource.rewrite.CollectionCleanRewrite;
import de.ids_mannheim.korap.resource.rewrite.DocMatchRewrite;
-//import de.ids_mannheim.korap.resource.rewrite.IdWriter;
+// import de.ids_mannheim.korap.resource.rewrite.IdWriter;
import de.ids_mannheim.korap.resource.rewrite.RewriteHandler;
-/** Defines rewrite handling methods relevant only in full version.
+/**
+ * Defines rewrite handling methods relevant only in full version.
*
* @author margaretha
*
*/
-public class FullRewriteHandler extends RewriteHandler{
+public class FullRewriteHandler extends RewriteHandler {
+ @Autowired
+ private VirtualCorpusRewrite vcRewrite;
+
public FullRewriteHandler (FullConfiguration config) {
super(config);
}
-
+
public void defaultRewriteConstraints () {
- this.add(CollectionRewrite.class);
-// this.add(IdWriter.class);
- this.add(DocMatchRewrite.class);
- this.add(CollectionCleanRewrite.class);
- }
+ this.addProcessor(vcRewrite);
+ this.add(CollectionRewrite.class);
+ // this.add(IdWriter.class);
+ this.add(DocMatchRewrite.class);
+ this.add(CollectionCleanRewrite.class);
+ }
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/rewrite/VirtualCorpusRewrite.java b/full/src/main/java/de/ids_mannheim/korap/rewrite/VirtualCorpusRewrite.java
new file mode 100644
index 0000000..ef7ec7d
--- /dev/null
+++ b/full/src/main/java/de/ids_mannheim/korap/rewrite/VirtualCorpusRewrite.java
@@ -0,0 +1,72 @@
+package de.ids_mannheim.korap.rewrite;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import de.ids_mannheim.korap.config.KustvaktConfiguration;
+import de.ids_mannheim.korap.entity.VirtualCorpus;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.resource.rewrite.KoralNode;
+import de.ids_mannheim.korap.resource.rewrite.KoralNode.RewriteIdentifier;
+import de.ids_mannheim.korap.resource.rewrite.RewriteTask;
+import de.ids_mannheim.korap.service.VirtualCorpusService;
+import de.ids_mannheim.korap.user.User;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+@Component
+public class VirtualCorpusRewrite implements RewriteTask.RewriteQuery {
+
+ @Autowired
+ private VirtualCorpusService vcService;
+
+ @Override
+ public JsonNode rewriteQuery (KoralNode node, KustvaktConfiguration config,
+ User user) throws KustvaktException {
+ if (node.has("collection")) {
+ findVCRef(user.getUsername(), node.at("/collection"));
+ }
+ return node.rawNode();
+ }
+
+ private void findVCRef (String username, KoralNode koralNode)
+ throws KustvaktException {
+ if (koralNode.has("@type")
+ && koralNode.get("@type").equals("koral:docGroupRef")) {
+ if (!koralNode.has("ref")) {
+ // throw error
+ }
+ else {
+ String vcName = koralNode.get("ref");
+
+ VirtualCorpus vc =
+ vcService.searchVCByName(username, vcName, "system");
+ if (!vc.isCached()) {
+ String koralQuery = vc.getKoralQuery();
+ JsonNode kq =
+ JsonUtils.readTree(koralQuery).at("/collection");
+ JsonNode jsonNode = koralNode.rawNode();
+ // rewrite
+ koralNode.remove("@type", new RewriteIdentifier("@type",
+ jsonNode.at("/@type").asText()));
+ koralNode.remove("ref", new RewriteIdentifier("ref",
+ jsonNode.at("/ref").asText()));
+ koralNode.setAll((ObjectNode) kq);
+ }
+ }
+
+ }
+ else if (koralNode.has("operands")) {
+ KoralNode operands = koralNode.at("/operands");
+
+ for (int i = 0; i < operands.size(); i++) {
+ KoralNode operand = operands.get(i);
+ findVCRef(username, operand);
+ operand.buildRewrites();
+ }
+
+ }
+ }
+}
diff --git a/full/src/main/java/de/ids_mannheim/korap/service/VirtualCorpusService.java b/full/src/main/java/de/ids_mannheim/korap/service/VirtualCorpusService.java
index 27a9c97..33af77b 100644
--- a/full/src/main/java/de/ids_mannheim/korap/service/VirtualCorpusService.java
+++ b/full/src/main/java/de/ids_mannheim/korap/service/VirtualCorpusService.java
@@ -55,7 +55,7 @@
private static Logger jlog =
LogManager.getLogger(VirtualCorpusService.class);
- public static Pattern wordPattern = Pattern.compile("[\\w ]+");
+ public static Pattern wordPattern = Pattern.compile("[-\\w ]+");
@Autowired
private VirtualCorpusDao vcDao;
@@ -123,7 +123,7 @@
Iterator<VirtualCorpus> i = vcList.iterator();
while (i.hasNext()) {
vc = i.next();
- String json = vc.getCorpusQuery();
+ String json = vc.getKoralQuery();
String statistics = krill.getStatistics(json);
VirtualCorpusDto vcDto =
converter.createVirtualCorpusDto(vc, statistics);
@@ -234,7 +234,8 @@
String name = vc.getName();
if (!wordPattern.matcher(name).matches()) {
throw new KustvaktException(StatusCodes.INVALID_ARGUMENT,
- "Virtual corpus name must only contains letters, numbers, underscores and spaces",
+ "Virtual corpus name must only contains letters, numbers, "
+ + "underscores, hypens and spaces",
name);
}
@@ -249,7 +250,7 @@
int vcId = vcDao.createVirtualCorpus(vc.getName(), vc.getType(),
requiredAccess, koralQuery, vc.getDefinition(),
- vc.getDescription(), vc.getStatus(), username);
+ vc.getDescription(), vc.getStatus(), vc.isCached(), username);
if (vc.getType().equals(VirtualCorpusType.PUBLISHED)) {
publishVC(vcId);
@@ -274,7 +275,7 @@
return koralQuery;
}
- private CorpusAccess determineRequiredAccess (String koralQuery)
+ public CorpusAccess determineRequiredAccess (String koralQuery)
throws KustvaktException {
if (findDocWithLicense(koralQuery, config.getAllOnlyRegex())) {
@@ -416,17 +417,32 @@
}
+ public VirtualCorpus searchVCByName (String username, String vcName, String createdBy)
+ throws KustvaktException {
+ VirtualCorpus vc = vcDao.retrieveVCByName(vcName, createdBy);
+ checkVCAccess(vc, username);
+ return vc;
+ }
+
public VirtualCorpusDto searchVCById (String username, int vcId)
throws KustvaktException {
VirtualCorpus vc = vcDao.retrieveVCById(vcId);
+ checkVCAccess(vc, username);
+ String json = vc.getKoralQuery();
+ String statistics = krill.getStatistics(json);
+ return converter.createVirtualCorpusDto(vc, statistics);
+ }
+
+ private void checkVCAccess (VirtualCorpus vc, String username)
+ throws KustvaktException {
VirtualCorpusType type = vc.getType();
if (!adminDao.isAdmin(username)
&& !username.equals(vc.getCreatedBy())) {
if (type.equals(VirtualCorpusType.PRIVATE)
|| (type.equals(VirtualCorpusType.PROJECT)
- && !hasAccess(username, vcId))) {
+ && !hasAccess(username, vc.getId()))) {
throw new KustvaktException(StatusCodes.AUTHORIZATION_FAILED,
"Unauthorized operation for user: " + username,
username);
@@ -434,8 +450,8 @@
else if (VirtualCorpusType.PUBLISHED.equals(type)) {
// add user in the VC's auto group
- UserGroup userGroup =
- userGroupService.retrieveHiddenUserGroupByVC(vcId);
+ UserGroup userGroup = userGroupService
+ .retrieveHiddenUserGroupByVC(vc.getId());
try {
userGroupService.addGroupMember(username, userGroup,
"system", GroupMemberStatus.ACTIVE);
@@ -448,10 +464,6 @@
}
// else VirtualCorpusType.SYSTEM
}
-
- String json = vc.getCorpusQuery();
- String statistics = krill.getStatistics(json);
- return converter.createVirtualCorpusDto(vc, statistics);
}
private boolean hasAccess (String username, int vcId)
diff --git a/full/src/main/java/de/ids_mannheim/korap/web/input/VirtualCorpusJson.java b/full/src/main/java/de/ids_mannheim/korap/web/input/VirtualCorpusJson.java
index a67876a..3d34d24 100644
--- a/full/src/main/java/de/ids_mannheim/korap/web/input/VirtualCorpusJson.java
+++ b/full/src/main/java/de/ids_mannheim/korap/web/input/VirtualCorpusJson.java
@@ -31,6 +31,7 @@
private String definition;
private String description;
private String status;
+ private boolean isCached;
public void setCorpusQuery (String corpusQuery)