Implemented VirtualCorpusRewrite.
Change-Id: I937cfa78db9be0602b17222f37de72cf3b63027c
diff --git a/core/Changes b/core/Changes
index ec10f8e..1b78700 100644
--- a/core/Changes
+++ b/core/Changes
@@ -1,7 +1,9 @@
version 0.61.2
12/09/2018
- Removed log4j loader and old OAuth2 implementation codes(margaretha)
-
+13/09/2018
+ - Updated KoralNode and enforced order of RewriteTask (margaretha)
+
version 0.61.1
28/08/2018
- Added API URL versioning (margaretha)
diff --git a/core/src/main/java/de/ids_mannheim/korap/resource/rewrite/KoralNode.java b/core/src/main/java/de/ids_mannheim/korap/resource/rewrite/KoralNode.java
index 70f23bc..46bb538 100644
--- a/core/src/main/java/de/ids_mannheim/korap/resource/rewrite/KoralNode.java
+++ b/core/src/main/java/de/ids_mannheim/korap/resource/rewrite/KoralNode.java
@@ -133,6 +133,13 @@
}
}
+ public void setAll (ObjectNode other) {
+ if (this.node.isObject()) {
+ ObjectNode n = (ObjectNode) this.node;
+ n.setAll(other);
+ }
+ this.rewrites.add("insertion",null);
+ }
public String get (String name) {
if (this.node.isObject())
@@ -204,7 +211,9 @@
public KoralRewriteBuilder add (String op, Object scope) {
KoralRewrite rewrite = new KoralRewrite();
rewrite.setOperation(op);
- rewrite.setScope(scope.toString());
+ if (scope !=null){
+ rewrite.setScope(scope.toString());
+ }
this.rewrites.add(rewrite);
return this;
}
@@ -270,4 +279,15 @@
public boolean isMissingNode (String string) {
return this.node.at(string).isMissingNode();
}
+
+
+ public int size () {
+ return this.node.size();
+ }
+
+
+ public KoralNode get (int i) {
+// this.node = this.node.get(i);
+ return this.wrapNode(this.node.get(i));
+ }
}
diff --git a/core/src/main/java/de/ids_mannheim/korap/resource/rewrite/RewriteHandler.java b/core/src/main/java/de/ids_mannheim/korap/resource/rewrite/RewriteHandler.java
index 592a144..a932fa9 100644
--- a/core/src/main/java/de/ids_mannheim/korap/resource/rewrite/RewriteHandler.java
+++ b/core/src/main/java/de/ids_mannheim/korap/resource/rewrite/RewriteHandler.java
@@ -5,6 +5,7 @@
import java.util.Collection;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
@@ -50,7 +51,7 @@
public RewriteHandler () {
this.node_processors = new HashSet<>();
this.token_node_processors = new HashSet<>();
- this.query_processors = new HashSet<>();
+ this.query_processors = new LinkedHashSet<>();
this.failed_task_registration = new HashSet<>();
this.beans = null;
this.add(FoundryInject.class);
diff --git a/core/src/main/java/de/ids_mannheim/korap/web/SearchKrill.java b/core/src/main/java/de/ids_mannheim/korap/web/SearchKrill.java
index 4317308..f52bbdb 100644
--- a/core/src/main/java/de/ids_mannheim/korap/web/SearchKrill.java
+++ b/core/src/main/java/de/ids_mannheim/korap/web/SearchKrill.java
@@ -36,7 +36,7 @@
String indexDir = "/data/prep_corpus/index/";
String i = "/Users/hanl/Projects/prep_corpus";
String klinux10 = "/vol/work/hanl/indices";
- private KrillIndex index;
+ public static KrillIndex index;
/**
* Constructor
diff --git a/full/Changes b/full/Changes
index 93798ca..252191b 100644
--- a/full/Changes
+++ b/full/Changes
@@ -1,6 +1,8 @@
# version 0.61.2
12/09/2018
- - Added various log4j2 configurations.
+ - Added various log4j2 configurations (margaretha)
+13/09/2018
+ - Implemented VirtualCorpusRewrite (margaretha)
# version 0.61.1
28/08/2018
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java b/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
index 57d78b2..85360c7 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/Initializator.java
@@ -5,6 +5,7 @@
import org.springframework.beans.factory.annotation.Autowired;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
import de.ids_mannheim.korap.oauth2.constant.OAuth2Scope;
import de.ids_mannheim.korap.oauth2.dao.AccessScopeDao;
import de.ids_mannheim.korap.util.QueryException;
@@ -25,7 +26,7 @@
public Initializator () {}
- public void init () throws IOException, QueryException {
+ public void init () throws IOException, QueryException, KustvaktException {
setInitialAccessScope();
loader.loadVCToCache();
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java b/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
index ff303b1..4bee9be 100644
--- a/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
+++ b/full/src/main/java/de/ids_mannheim/korap/config/NamedVCLoader.java
@@ -13,6 +13,12 @@
import org.springframework.stereotype.Component;
import de.ids_mannheim.korap.KrillCollection;
+import de.ids_mannheim.korap.constant.VirtualCorpusType;
+import de.ids_mannheim.korap.dao.VirtualCorpusDao;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.exceptions.StatusCodes;
+import de.ids_mannheim.korap.service.VirtualCorpusService;
+import de.ids_mannheim.korap.user.User.CorpusAccess;
import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.web.SearchKrill;
@@ -22,10 +28,14 @@
private FullConfiguration config;
@Autowired
private SearchKrill searchKrill;
-
+ @Autowired
+ private VirtualCorpusDao vcDao;
+ @Autowired
+ private VirtualCorpusService vcService;
+
private static Logger jlog = LogManager.getLogger(NamedVCLoader.class);
- public void loadVCToCache () throws IOException, QueryException {
+ public void loadVCToCache () throws IOException, QueryException, KustvaktException {
String dir = config.getNamedVCPath();
File d = new File(dir);
@@ -66,18 +76,39 @@
continue;
}
jlog.debug(
- "READ " + file.getName() + " duration: " + (end - start));
+ "READ " + filename + " duration: " + (end - start));
- KrillCollection collection = new KrillCollection(json);
- collection.setIndex(searchKrill.getIndex());
-
- if (collection != null) {
- collection.storeInCache(filename);
- }
- end = System.currentTimeMillis();
- jlog.debug(filename + "caching duration: " + (end - start));
- jlog.debug("memory cache: "
- + KrillCollection.cache.calculateInMemorySize());
+ cacheVC(json, filename);
+ storeVC(filename, json);
}
}
+
+ private void cacheVC (String json, String filename) throws IOException, QueryException {
+ long start, end;
+ start = System.currentTimeMillis();
+
+ KrillCollection collection = new KrillCollection(json);
+ collection.setIndex(searchKrill.getIndex());
+
+ if (collection != null) {
+ collection.storeInCache(filename);
+ }
+ end = System.currentTimeMillis();
+ jlog.info(filename + " caching duration: " + (end - start));
+ jlog.debug("memory cache: "
+ + KrillCollection.cache.calculateInMemorySize());
+ }
+
+ private void storeVC (String name, String koralQuery) throws KustvaktException {
+ if (!VirtualCorpusService.wordPattern.matcher(name).matches()) {
+ throw new KustvaktException(StatusCodes.INVALID_ARGUMENT,
+ "Virtual corpus name must only contains letters, numbers, "
+ + "underscores, hypens and spaces",
+ name);
+ }
+ CorpusAccess requiredAccess = vcService.determineRequiredAccess(koralQuery);
+ vcDao.createVirtualCorpus(name, VirtualCorpusType.SYSTEM,
+ requiredAccess, koralQuery, null,
+ null, null, true, "system");
+ }
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/dao/VirtualCorpusDao.java b/full/src/main/java/de/ids_mannheim/korap/dao/VirtualCorpusDao.java
index 9e9983c..c8006aa 100644
--- a/full/src/main/java/de/ids_mannheim/korap/dao/VirtualCorpusDao.java
+++ b/full/src/main/java/de/ids_mannheim/korap/dao/VirtualCorpusDao.java
@@ -36,9 +36,10 @@
import de.ids_mannheim.korap.user.User.CorpusAccess;
import de.ids_mannheim.korap.utils.ParameterChecker;
-/** VirtualCorpusDao manages SQL queries regarding virtual corpora,
- * e.g. retrieving and storing virtual corpora.
- *
+/**
+ * VirtualCorpusDao manages SQL queries regarding virtual corpora,
+ * e.g. retrieving and storing virtual corpora.
+ *
* @author margaretha
*
*/
@@ -50,19 +51,20 @@
private EntityManager entityManager;
public int createVirtualCorpus (String name, VirtualCorpusType type,
- CorpusAccess requiredAccess, String corpusQuery, String definition,
- String description, String status, String createdBy)
- throws KustvaktException {
+ CorpusAccess requiredAccess, String koralQuery, String definition,
+ String description, String status, boolean isCached,
+ String createdBy) throws KustvaktException {
VirtualCorpus vc = new VirtualCorpus();
vc.setName(name);
vc.setType(type);
vc.setRequiredAccess(requiredAccess);
- vc.setCorpusQuery(corpusQuery);
+ vc.setKoralQuery(koralQuery);
vc.setDefinition(definition);
vc.setDescription(description);
vc.setStatus(status);
vc.setCreatedBy(createdBy);
+ vc.setCached(isCached);
entityManager.persist(vc);
return vc.getId();
@@ -70,7 +72,7 @@
public void editVirtualCorpus (VirtualCorpus vc, String name,
VirtualCorpusType type, CorpusAccess requiredAccess,
- String corpusQuery, String definition, String description,
+ String koralQuery, String definition, String description,
String status) throws KustvaktException {
if (name != null && !name.isEmpty()) {
@@ -82,8 +84,8 @@
if (requiredAccess != null) {
vc.setRequiredAccess(requiredAccess);
}
- if (corpusQuery != null) {
- vc.setCorpusQuery(corpusQuery);
+ if (koralQuery != null) {
+ vc.setKoralQuery(koralQuery);
}
if (definition != null && !definition.isEmpty()) {
vc.setDefinition(definition);
@@ -97,22 +99,28 @@
entityManager.merge(vc);
}
- public void deleteVirtualCorpus (VirtualCorpus vc) throws KustvaktException {
- if (!entityManager.contains(vc)){
+ public void deleteVirtualCorpus (VirtualCorpus vc)
+ throws KustvaktException {
+ if (!entityManager.contains(vc)) {
vc = entityManager.merge(vc);
}
entityManager.remove(vc);
-
+
}
- /** System admin function.
+ /**
+ * System admin function.
*
- * Retrieves virtual corpus by creator and type. If type is not specified,
- * retrieves virtual corpora of all types. If createdBy is not specified,
- * retrieves virtual corpora of all users.
+ * Retrieves virtual corpus by creator and type. If type is not
+ * specified,
+ * retrieves virtual corpora of all types. If createdBy is not
+ * specified,
+ * retrieves virtual corpora of all users.
*
- * @param type {@link VirtualCorpusType}
- * @param createdBy username of the virtual corpus creator
+ * @param type
+ * {@link VirtualCorpusType}
+ * @param createdBy
+ * username of the virtual corpus creator
* @return a list of {@link VirtualCorpus}
* @throws KustvaktException
*/
@@ -172,6 +180,40 @@
return vc;
}
+ public VirtualCorpus retrieveVCByName (String vcName, String createdBy)
+ throws KustvaktException {
+ ParameterChecker.checkStringValue(createdBy, "createdBy");
+ ParameterChecker.checkStringValue(vcName, "vcName");
+
+ CriteriaBuilder builder = entityManager.getCriteriaBuilder();
+ CriteriaQuery<VirtualCorpus> query =
+ builder.createQuery(VirtualCorpus.class);
+
+ Root<VirtualCorpus> virtualCorpus = query.from(VirtualCorpus.class);
+
+ Predicate condition = builder.and(
+ builder.equal(virtualCorpus.get(VirtualCorpus_.createdBy),
+ createdBy),
+ builder.equal(virtualCorpus.get(VirtualCorpus_.name), vcName));
+
+ query.select(virtualCorpus);
+ query.where(condition);
+
+ Query q = entityManager.createQuery(query);
+ VirtualCorpus vc = null;
+ try {
+ vc = (VirtualCorpus) q.getSingleResult();
+ }
+ catch (NoResultException e) {
+ String vcCode = createdBy + "/" + vcName;
+ throw new KustvaktException(StatusCodes.NO_RESULT_FOUND,
+ "No result found for query: retrieve virtual corpus by name "
+ + vcCode,
+ String.valueOf(vcCode), e);
+ }
+ return vc;
+ }
+
@SuppressWarnings("unchecked")
public List<VirtualCorpus> retrieveOwnerVC (String userId)
throws KustvaktException {
@@ -225,11 +267,11 @@
Join<VirtualCorpus, VirtualCorpusAccess> access =
virtualCorpus.join(VirtualCorpus_.virtualCorpusAccess);
- // Predicate corpusStatus = builder.and(
- // builder.notEqual(access.get(VirtualCorpusAccess_.status),
- // VirtualCorpusAccessStatus.HIDDEN),
- // builder.notEqual(access.get(VirtualCorpusAccess_.status),
- // VirtualCorpusAccessStatus.DELETED));
+ // Predicate corpusStatus = builder.and(
+ // builder.notEqual(access.get(VirtualCorpusAccess_.status),
+ // VirtualCorpusAccessStatus.HIDDEN),
+ // builder.notEqual(access.get(VirtualCorpusAccess_.status),
+ // VirtualCorpusAccessStatus.DELETED));
Predicate corpusStatus =
builder.notEqual(access.get(VirtualCorpusAccess_.status),
@@ -270,7 +312,6 @@
builder.equal(virtualCorpus.get(VirtualCorpus_.type),
VirtualCorpusType.SYSTEM));
-
query.select(virtualCorpus);
query.where(predicate);
query.distinct(true);
diff --git a/full/src/main/java/de/ids_mannheim/korap/dto/converter/VirtualCorpusConverter.java b/full/src/main/java/de/ids_mannheim/korap/dto/converter/VirtualCorpusConverter.java
index 52cbfe6..9fec23d 100644
--- a/full/src/main/java/de/ids_mannheim/korap/dto/converter/VirtualCorpusConverter.java
+++ b/full/src/main/java/de/ids_mannheim/korap/dto/converter/VirtualCorpusConverter.java
@@ -31,7 +31,7 @@
dto.setStatus(vc.getStatus());
dto.setDescription(vc.getDescription());
dto.setType(vc.getType().displayName());
- dto.setKoralQuery(vc.getCorpusQuery());
+ dto.setKoralQuery(vc.getKoralQuery());
JsonNode node = JsonUtils.readTree(statistics);
int numberOfDoc = node.at("/documents").asInt();
diff --git a/full/src/main/java/de/ids_mannheim/korap/entity/VirtualCorpus.java b/full/src/main/java/de/ids_mannheim/korap/entity/VirtualCorpus.java
index 78f9fff..4a6aeeb 100644
--- a/full/src/main/java/de/ids_mannheim/korap/entity/VirtualCorpus.java
+++ b/full/src/main/java/de/ids_mannheim/korap/entity/VirtualCorpus.java
@@ -48,10 +48,12 @@
@Column(name = "required_access")
private CorpusAccess requiredAccess;
@Column(name = "corpus_query")
- private String corpusQuery;
+ private String koralQuery;
private String definition;
@Column(name = "created_by")
private String createdBy;
+ @Column(name = "is_cached")
+ private boolean isCached;
@OneToMany(mappedBy = "virtualCorpus", fetch = FetchType.LAZY,
cascade = CascadeType.REMOVE)
@@ -61,7 +63,7 @@
public String toString () {
return "id=" + id + ", name= " + name + ", type= " + type + ", status= "
+ status + ", description=" + description + ", requiredAccess="
- + requiredAccess + ", corpusQuery= " + corpusQuery
+ + requiredAccess + ", corpusQuery= " + koralQuery
+ ", definition= " + definition + ", createdBy= " + createdBy;
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/rewrite/CollectionRewrite.java b/full/src/main/java/de/ids_mannheim/korap/rewrite/CollectionRewrite.java
index 08c252f..091c1a8 100644
--- a/full/src/main/java/de/ids_mannheim/korap/rewrite/CollectionRewrite.java
+++ b/full/src/main/java/de/ids_mannheim/korap/rewrite/CollectionRewrite.java
@@ -144,7 +144,7 @@
KoralCollectionQueryBuilder builder = new KoralCollectionQueryBuilder();
RewriteIdentifier identifier = new KoralNode.RewriteIdentifier(
Attributes.AVAILABILITY, user.getCorpusAccess());
- JsonNode rewrittesNode;
+ JsonNode rewrittenNode;
if (jsonNode.has("collection")) {
List<String> avalabilityCopy =
@@ -159,16 +159,16 @@
builder.with(buildAvailability(avalabilityCopy));
jlog.debug("corpus query: " + builder.toString());
builder.setBaseQuery(builder.toJSON());
- rewrittesNode = builder.mergeWith(jsonNode).at("/collection");
- node.set("collection", rewrittesNode, identifier);
+ rewrittenNode = builder.mergeWith(jsonNode).at("/collection");
+ node.set("collection", rewrittenNode, identifier);
}
}
else {
builder.with(buildAvailability(userAvailabilities));
jlog.debug("corpus query: " + builder.toString());
- rewrittesNode =
+ rewrittenNode =
JsonUtils.readTree(builder.toJSON()).at("/collection");
- node.set("collection", rewrittesNode, identifier);
+ node.set("collection", rewrittenNode, identifier);
}
jlog.debug("REWRITES: " + node.at("/collection").toString());
diff --git a/full/src/main/java/de/ids_mannheim/korap/rewrite/FullRewriteHandler.java b/full/src/main/java/de/ids_mannheim/korap/rewrite/FullRewriteHandler.java
index 0926a09..21bcc41 100644
--- a/full/src/main/java/de/ids_mannheim/korap/rewrite/FullRewriteHandler.java
+++ b/full/src/main/java/de/ids_mannheim/korap/rewrite/FullRewriteHandler.java
@@ -1,26 +1,34 @@
package de.ids_mannheim.korap.rewrite;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
import de.ids_mannheim.korap.config.FullConfiguration;
import de.ids_mannheim.korap.resource.rewrite.CollectionCleanRewrite;
import de.ids_mannheim.korap.resource.rewrite.DocMatchRewrite;
-//import de.ids_mannheim.korap.resource.rewrite.IdWriter;
+// import de.ids_mannheim.korap.resource.rewrite.IdWriter;
import de.ids_mannheim.korap.resource.rewrite.RewriteHandler;
-/** Defines rewrite handling methods relevant only in full version.
+/**
+ * Defines rewrite handling methods relevant only in full version.
*
* @author margaretha
*
*/
-public class FullRewriteHandler extends RewriteHandler{
+public class FullRewriteHandler extends RewriteHandler {
+ @Autowired
+ private VirtualCorpusRewrite vcRewrite;
+
public FullRewriteHandler (FullConfiguration config) {
super(config);
}
-
+
public void defaultRewriteConstraints () {
- this.add(CollectionRewrite.class);
-// this.add(IdWriter.class);
- this.add(DocMatchRewrite.class);
- this.add(CollectionCleanRewrite.class);
- }
+ this.addProcessor(vcRewrite);
+ this.add(CollectionRewrite.class);
+ // this.add(IdWriter.class);
+ this.add(DocMatchRewrite.class);
+ this.add(CollectionCleanRewrite.class);
+ }
}
diff --git a/full/src/main/java/de/ids_mannheim/korap/rewrite/VirtualCorpusRewrite.java b/full/src/main/java/de/ids_mannheim/korap/rewrite/VirtualCorpusRewrite.java
new file mode 100644
index 0000000..ef7ec7d
--- /dev/null
+++ b/full/src/main/java/de/ids_mannheim/korap/rewrite/VirtualCorpusRewrite.java
@@ -0,0 +1,72 @@
+package de.ids_mannheim.korap.rewrite;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import de.ids_mannheim.korap.config.KustvaktConfiguration;
+import de.ids_mannheim.korap.entity.VirtualCorpus;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.resource.rewrite.KoralNode;
+import de.ids_mannheim.korap.resource.rewrite.KoralNode.RewriteIdentifier;
+import de.ids_mannheim.korap.resource.rewrite.RewriteTask;
+import de.ids_mannheim.korap.service.VirtualCorpusService;
+import de.ids_mannheim.korap.user.User;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+@Component
+public class VirtualCorpusRewrite implements RewriteTask.RewriteQuery {
+
+ @Autowired
+ private VirtualCorpusService vcService;
+
+ @Override
+ public JsonNode rewriteQuery (KoralNode node, KustvaktConfiguration config,
+ User user) throws KustvaktException {
+ if (node.has("collection")) {
+ findVCRef(user.getUsername(), node.at("/collection"));
+ }
+ return node.rawNode();
+ }
+
+ private void findVCRef (String username, KoralNode koralNode)
+ throws KustvaktException {
+ if (koralNode.has("@type")
+ && koralNode.get("@type").equals("koral:docGroupRef")) {
+ if (!koralNode.has("ref")) {
+ // throw error
+ }
+ else {
+ String vcName = koralNode.get("ref");
+
+ VirtualCorpus vc =
+ vcService.searchVCByName(username, vcName, "system");
+ if (!vc.isCached()) {
+ String koralQuery = vc.getKoralQuery();
+ JsonNode kq =
+ JsonUtils.readTree(koralQuery).at("/collection");
+ JsonNode jsonNode = koralNode.rawNode();
+ // rewrite
+ koralNode.remove("@type", new RewriteIdentifier("@type",
+ jsonNode.at("/@type").asText()));
+ koralNode.remove("ref", new RewriteIdentifier("ref",
+ jsonNode.at("/ref").asText()));
+ koralNode.setAll((ObjectNode) kq);
+ }
+ }
+
+ }
+ else if (koralNode.has("operands")) {
+ KoralNode operands = koralNode.at("/operands");
+
+ for (int i = 0; i < operands.size(); i++) {
+ KoralNode operand = operands.get(i);
+ findVCRef(username, operand);
+ operand.buildRewrites();
+ }
+
+ }
+ }
+}
diff --git a/full/src/main/java/de/ids_mannheim/korap/service/VirtualCorpusService.java b/full/src/main/java/de/ids_mannheim/korap/service/VirtualCorpusService.java
index 27a9c97..33af77b 100644
--- a/full/src/main/java/de/ids_mannheim/korap/service/VirtualCorpusService.java
+++ b/full/src/main/java/de/ids_mannheim/korap/service/VirtualCorpusService.java
@@ -55,7 +55,7 @@
private static Logger jlog =
LogManager.getLogger(VirtualCorpusService.class);
- public static Pattern wordPattern = Pattern.compile("[\\w ]+");
+ public static Pattern wordPattern = Pattern.compile("[-\\w ]+");
@Autowired
private VirtualCorpusDao vcDao;
@@ -123,7 +123,7 @@
Iterator<VirtualCorpus> i = vcList.iterator();
while (i.hasNext()) {
vc = i.next();
- String json = vc.getCorpusQuery();
+ String json = vc.getKoralQuery();
String statistics = krill.getStatistics(json);
VirtualCorpusDto vcDto =
converter.createVirtualCorpusDto(vc, statistics);
@@ -234,7 +234,8 @@
String name = vc.getName();
if (!wordPattern.matcher(name).matches()) {
throw new KustvaktException(StatusCodes.INVALID_ARGUMENT,
- "Virtual corpus name must only contains letters, numbers, underscores and spaces",
+ "Virtual corpus name must only contains letters, numbers, "
+ + "underscores, hypens and spaces",
name);
}
@@ -249,7 +250,7 @@
int vcId = vcDao.createVirtualCorpus(vc.getName(), vc.getType(),
requiredAccess, koralQuery, vc.getDefinition(),
- vc.getDescription(), vc.getStatus(), username);
+ vc.getDescription(), vc.getStatus(), vc.isCached(), username);
if (vc.getType().equals(VirtualCorpusType.PUBLISHED)) {
publishVC(vcId);
@@ -274,7 +275,7 @@
return koralQuery;
}
- private CorpusAccess determineRequiredAccess (String koralQuery)
+ public CorpusAccess determineRequiredAccess (String koralQuery)
throws KustvaktException {
if (findDocWithLicense(koralQuery, config.getAllOnlyRegex())) {
@@ -416,17 +417,32 @@
}
+ public VirtualCorpus searchVCByName (String username, String vcName, String createdBy)
+ throws KustvaktException {
+ VirtualCorpus vc = vcDao.retrieveVCByName(vcName, createdBy);
+ checkVCAccess(vc, username);
+ return vc;
+ }
+
public VirtualCorpusDto searchVCById (String username, int vcId)
throws KustvaktException {
VirtualCorpus vc = vcDao.retrieveVCById(vcId);
+ checkVCAccess(vc, username);
+ String json = vc.getKoralQuery();
+ String statistics = krill.getStatistics(json);
+ return converter.createVirtualCorpusDto(vc, statistics);
+ }
+
+ private void checkVCAccess (VirtualCorpus vc, String username)
+ throws KustvaktException {
VirtualCorpusType type = vc.getType();
if (!adminDao.isAdmin(username)
&& !username.equals(vc.getCreatedBy())) {
if (type.equals(VirtualCorpusType.PRIVATE)
|| (type.equals(VirtualCorpusType.PROJECT)
- && !hasAccess(username, vcId))) {
+ && !hasAccess(username, vc.getId()))) {
throw new KustvaktException(StatusCodes.AUTHORIZATION_FAILED,
"Unauthorized operation for user: " + username,
username);
@@ -434,8 +450,8 @@
else if (VirtualCorpusType.PUBLISHED.equals(type)) {
// add user in the VC's auto group
- UserGroup userGroup =
- userGroupService.retrieveHiddenUserGroupByVC(vcId);
+ UserGroup userGroup = userGroupService
+ .retrieveHiddenUserGroupByVC(vc.getId());
try {
userGroupService.addGroupMember(username, userGroup,
"system", GroupMemberStatus.ACTIVE);
@@ -448,10 +464,6 @@
}
// else VirtualCorpusType.SYSTEM
}
-
- String json = vc.getCorpusQuery();
- String statistics = krill.getStatistics(json);
- return converter.createVirtualCorpusDto(vc, statistics);
}
private boolean hasAccess (String username, int vcId)
diff --git a/full/src/main/java/de/ids_mannheim/korap/web/input/VirtualCorpusJson.java b/full/src/main/java/de/ids_mannheim/korap/web/input/VirtualCorpusJson.java
index a67876a..3d34d24 100644
--- a/full/src/main/java/de/ids_mannheim/korap/web/input/VirtualCorpusJson.java
+++ b/full/src/main/java/de/ids_mannheim/korap/web/input/VirtualCorpusJson.java
@@ -31,6 +31,7 @@
private String definition;
private String description;
private String status;
+ private boolean isCached;
public void setCorpusQuery (String corpusQuery)
diff --git a/full/src/main/resources/db/new-mysql/V1.1__create_virtual_corpus_tables.sql b/full/src/main/resources/db/new-mysql/V1.1__create_virtual_corpus_tables.sql
index 3b8022b..55f2163 100644
--- a/full/src/main/resources/db/new-mysql/V1.1__create_virtual_corpus_tables.sql
+++ b/full/src/main/resources/db/new-mysql/V1.1__create_virtual_corpus_tables.sql
@@ -63,6 +63,7 @@
status varchar(100) DEFAULT NULL,
corpus_query TEXT NOT NULL,
definition varchar(255) DEFAULT NULL,
+ is_cached BOOLEAN DEFAULT 0,
INDEX owner_index (created_by),
INDEX type_index (type)
);
diff --git a/full/src/main/resources/db/new-sqlite/V1.1__create_virtual_corpus_tables.sql b/full/src/main/resources/db/new-sqlite/V1.1__create_virtual_corpus_tables.sql
index 462e85a..56fc105 100644
--- a/full/src/main/resources/db/new-sqlite/V1.1__create_virtual_corpus_tables.sql
+++ b/full/src/main/resources/db/new-sqlite/V1.1__create_virtual_corpus_tables.sql
@@ -72,7 +72,8 @@
description VARCHAR(255) DEFAULT NULL,
status VARCHAR(100) DEFAULT NULL,
corpus_query TEXT NOT NULL,
- definition VARCHAR(255) DEFAULT NULL
+ definition VARCHAR(255) DEFAULT NULL,
+ is_cached BOOLEAN DEFAULT 0
);
CREATE INDEX virtual_corpus_owner_index ON virtual_corpus(created_by);
diff --git a/full/src/test/java/de/ids_mannheim/korap/config/NamedVCLoaderTest.java b/full/src/test/java/de/ids_mannheim/korap/config/NamedVCLoaderTest.java
new file mode 100644
index 0000000..b5b218a
--- /dev/null
+++ b/full/src/test/java/de/ids_mannheim/korap/config/NamedVCLoaderTest.java
@@ -0,0 +1,35 @@
+package de.ids_mannheim.korap.config;
+
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+
+import org.junit.Test;
+import org.springframework.beans.factory.annotation.Autowired;
+
+import de.ids_mannheim.korap.KrillCollection;
+import de.ids_mannheim.korap.collection.CachedVCData;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.util.QueryException;
+import net.sf.ehcache.Element;
+
+public class NamedVCLoaderTest extends SpringJerseyTest{
+
+ @Autowired
+ private NamedVCLoader vcLoader;
+
+
+ @Test
+ public void testNamedVCLoader () throws IOException, QueryException, KustvaktException {
+ Element element = KrillCollection.cache.get("named-vc1");
+ assertTrue(element==null);
+
+ vcLoader.loadVCToCache();
+
+ element = KrillCollection.cache.get("named-vc1");
+ assertNotNull(element);
+ CachedVCData cachedData = (CachedVCData) element.getObjectValue();
+ assertTrue(cachedData.getDocIdMap().size() > 0);
+ }
+}
diff --git a/full/src/test/java/de/ids_mannheim/korap/dao/UserGroupDaoTest.java b/full/src/test/java/de/ids_mannheim/korap/dao/UserGroupDaoTest.java
index 40d4abd..1cdd7a5 100644
--- a/full/src/test/java/de/ids_mannheim/korap/dao/UserGroupDaoTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/dao/UserGroupDaoTest.java
@@ -146,7 +146,7 @@
String name = "dory new vc";
int id = virtualCorpusDao.createVirtualCorpus(name,
VirtualCorpusType.PROJECT, CorpusAccess.PUB,
- "corpusSigle=WPD15", "", "", "", createdBy);
+ "corpusSigle=WPD15", "", "", "", false, createdBy);
VirtualCorpus virtualCorpus = virtualCorpusDao.retrieveVCById(id);
userGroupDao.addVCToGroup(virtualCorpus, createdBy,
diff --git a/full/src/test/java/de/ids_mannheim/korap/dao/VirtualCorpusDaoTest.java b/full/src/test/java/de/ids_mannheim/korap/dao/VirtualCorpusDaoTest.java
index 3b97caa..e15a518 100644
--- a/full/src/test/java/de/ids_mannheim/korap/dao/VirtualCorpusDaoTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/dao/VirtualCorpusDaoTest.java
@@ -45,7 +45,7 @@
// insert vc
int id = dao.createVirtualCorpus("system VC", VirtualCorpusType.SYSTEM,
User.CorpusAccess.FREE, "corpusSigle=GOE", "definition",
- "description", "experimental", "test class");
+ "description", "experimental", false, "test class");
// select vc
List<VirtualCorpus> vcList =
diff --git a/full/src/test/java/de/ids_mannheim/korap/resource/rewrite/VirtualCorpusRewriteTest.java b/full/src/test/java/de/ids_mannheim/korap/resource/rewrite/VirtualCorpusRewriteTest.java
new file mode 100644
index 0000000..a388ae1
--- /dev/null
+++ b/full/src/test/java/de/ids_mannheim/korap/resource/rewrite/VirtualCorpusRewriteTest.java
@@ -0,0 +1,62 @@
+package de.ids_mannheim.korap.resource.rewrite;
+
+import static org.junit.Assert.assertEquals;
+
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.google.common.net.HttpHeaders;
+import com.sun.jersey.api.client.ClientResponse;
+
+import de.ids_mannheim.korap.authentication.http.HttpAuthorizationHandler;
+import de.ids_mannheim.korap.config.Attributes;
+import de.ids_mannheim.korap.config.SpringJerseyTest;
+import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.utils.JsonUtils;
+
+/**
+ * @author margaretha
+ *
+ */
+public class VirtualCorpusRewriteTest extends SpringJerseyTest {
+
+ @Test
+ public void testSearchFreeWithVCRef () throws KustvaktException {
+ ClientResponse response = resource().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("cq", "referTo \"system VC\"")
+ .get(ClientResponse.class);
+
+ String ent = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(ent);
+ node = node.at("/collection");
+
+ assertEquals("koral:docGroup", node.at("/@type").asText());
+ assertEquals(3, node.at("/operands/1/rewrites").size());
+ node = node.at("/operands/1/rewrites");
+ assertEquals("operation:deletion", node.at("/0/operation").asText());
+ assertEquals("operation:deletion", node.at("/1/operation").asText());
+ assertEquals("operation:insertion", node.at("/2/operation").asText());
+ }
+
+ @Test
+ public void testSearchPubWithVCRef () throws KustvaktException {
+ ClientResponse response = resource().path(API_VERSION).path("search")
+ .queryParam("q", "[orth=der]").queryParam("ql", "poliqarp")
+ .queryParam("cq", "referTo \"system VC\"")
+ .header(Attributes.AUTHORIZATION, HttpAuthorizationHandler
+ .createBasicAuthorizationHeaderValue("user", "pass"))
+ .header(HttpHeaders.X_FORWARDED_FOR, "149.27.0.32")
+ .get(ClientResponse.class);
+
+ String ent = response.getEntity(String.class);
+ JsonNode node = JsonUtils.readTree(ent);
+ node = node.at("/collection");
+ assertEquals("koral:docGroup", node.at("/@type").asText());
+ assertEquals("koral:docGroup", node.at("/operands/0/@type").asText());
+ assertEquals(3, node.at("/operands/1/rewrites").size());
+ assertEquals("koral:doc", node.at("/operands/1/@type").asText());
+ assertEquals("GOE", node.at("/operands/1/value").asText());
+ assertEquals("corpusSigle", node.at("/operands/1/key").asText());
+ }
+}
diff --git a/full/src/test/java/de/ids_mannheim/korap/web/controller/VCReferenceTest.java b/full/src/test/java/de/ids_mannheim/korap/web/controller/VCReferenceTest.java
index a7e2195..4ce02ff 100644
--- a/full/src/test/java/de/ids_mannheim/korap/web/controller/VCReferenceTest.java
+++ b/full/src/test/java/de/ids_mannheim/korap/web/controller/VCReferenceTest.java
@@ -1,8 +1,12 @@
package de.ids_mannheim.korap.web.controller;
import static org.junit.Assert.assertEquals;
+
import static org.junit.Assert.assertTrue;
+import java.io.IOException;
+
+import org.junit.Ignore;
import org.junit.Test;
import com.fasterxml.jackson.databind.JsonNode;
@@ -10,14 +14,19 @@
import de.ids_mannheim.korap.config.SpringJerseyTest;
import de.ids_mannheim.korap.exceptions.KustvaktException;
+import de.ids_mannheim.korap.util.QueryException;
import de.ids_mannheim.korap.utils.JsonUtils;
public class VCReferenceTest extends SpringJerseyTest {
+ // auto caching is disabled in krill
+ @Ignore
@Test
- public void testVCRef () throws KustvaktException {
+ public void testVCRef ()
+ throws KustvaktException, IOException, QueryException {
testSearchWithoutVCRefOr();
testSearchWithoutVCRefAnd();
+
// auto caching
testStatisticsWithVCReference();
testSearchWithVCRefNotEqual();
@@ -75,13 +84,12 @@
public void testStatisticsWithVCReference () throws KustvaktException {
String corpusQuery = "availability = /CC-BY.*/ & referTo named-vc1";
- ClientResponse response = resource().path(API_VERSION).path("statistics")
- .queryParam("corpusQuery", corpusQuery)
+ ClientResponse response = resource().path(API_VERSION)
+ .path("statistics").queryParam("corpusQuery", corpusQuery)
.get(ClientResponse.class);
String ent = response.getEntity(String.class);
JsonNode node = JsonUtils.readTree(ent);
assertEquals(2, node.at("/documents").asInt());
}
-
}