Implemented search for protected resources
by using public metadata search web-service and adding Diagnostics.
(#64)
Change-Id: Ia8853d50eb01168c86cb582943022493b9a16ff8
diff --git a/ChangeLog b/ChangeLog
index 70ec602..2221304 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -2,8 +2,9 @@
- handle resource pid and resolve resources (virtual corpora) using map (#62)
- handle empty description.
-- Add cache for virtual corpus access.
-
+- Add cache for virtual corpus access.(#63)
+- Implemented search for protected resources by using public metadata search
+ web-service and adding Diagnostics. (#64)
1.0.7-SNAPSHOT
diff --git a/src/main/java/de/ids_mannheim/korap/sru/KorapClient.java b/src/main/java/de/ids_mannheim/korap/sru/KorapClient.java
index b86739d..25fe426 100644
--- a/src/main/java/de/ids_mannheim/korap/sru/KorapClient.java
+++ b/src/main/java/de/ids_mannheim/korap/sru/KorapClient.java
@@ -34,6 +34,7 @@
import de.ids_mannheim.korap.util.RedirectStrategy;
import eu.clarin.sru.server.SRUConstants;
+import eu.clarin.sru.server.SRUDiagnosticList;
import eu.clarin.sru.server.SRUException;
/**
@@ -121,7 +122,8 @@
for (KorapResource r : resources) {
String[] urlParts = r.getLandingPage().split("cq=");
boolean freeAccess = false;
- if (r.getRequiredAccess().equals("FREE")) {
+ if (r.getRequiredAccess() != null &&
+ r.getRequiredAccess().equals("FREE")) {
freeAccess = true;
}
if (urlParts.length > 1 && !urlParts[1].isEmpty()) {
@@ -161,6 +163,7 @@
* the number of maximum records/matches to retrieve
* @param corpora
* the corpora to search on
+ * @param diagnostics
* @return a KorapResult
*
* @throws HttpResponseException
@@ -169,7 +172,8 @@
*/
public KorapResult query (String query, QueryLanguage queryLanguage,
String version, int startRecord, int maximumRecords,
- String[] corpora) throws IOException, SRUException {
+ String[] corpora, SRUDiagnosticList diagnostics) throws
+ IOException, SRUException {
if (query == null) {
throw new NullPointerException("Query is null.");
@@ -184,10 +188,15 @@
throw new IllegalArgumentException("Maximum records is too low.");
}
+ boolean freeAccess = true;
+
HttpUriRequest httpRequest = null;
try {
+ String corpusQuery = obtainCorpusQuery(corpora);
+ freeAccess = isAccessFree(corpora);
httpRequest = createSearchRequest(query, queryLanguage, version,
- startRecord - 1, maximumRecords, corpora);
+ startRecord - 1, maximumRecords, corpusQuery,
+ freeAccess);
}
catch (URISyntaxException e) {
throw new IOException("Failed creating http request.");
@@ -196,7 +205,7 @@
CloseableHttpResponse response = null;
KorapResult result = null;
try {
- response = sendRequest(httpRequest);
+ response = sendRequest(httpRequest, freeAccess, diagnostics);
// logger.info(response.toString());
BufferedInputStream jsonStream =
@@ -214,11 +223,59 @@
}
}
+ // remove matches for public search metadata response
+ if (!freeAccess) {
+ result.getMatches().clear();
+ }
+
return result;
}
- private CloseableHttpResponse sendRequest (HttpUriRequest httpRequest)
- throws ClientProtocolException, IOException {
+ private String obtainCorpusQuery (String[] corpora)
+ throws URISyntaxException, IOException, SRUException {
+ String corpusQuery = "";
+ if (corpora != null && corpora.length > 0) {
+ for (int i = 0; i < corpora.length; i++) {
+ String pid = corpora[i];
+ String cq = virtualCorpusQueries.get(pid);
+
+ if (cq == null) {
+ retrieveResources();
+ cq = virtualCorpusQueries.get(pid);
+ if (cq == null) {
+ throw new SRUException(
+ SRUConstants.SRU_GENERAL_SYSTEM_ERROR,
+ "Virtual corpus with pid: " + pid
+ + " is not found.");
+ }
+ }
+ cq = URLDecoder.decode(cq, "utf-8");
+ if (i == 0) {
+ corpusQuery = cq;
+ }
+ else {
+ corpusQuery += " & " + cq;
+ }
+ }
+ }
+ return corpusQuery;
+ }
+
+ private boolean isAccessFree (String[] corpora) {
+ if (corpora != null && corpora.length > 0) {
+ for (int i = 0; i < corpora.length; i++) {
+ String pid = corpora[i];
+ if (!virtualCorpusAccesses.get(pid)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private CloseableHttpResponse sendRequest (HttpUriRequest httpRequest,
+ boolean freeAccess, SRUDiagnosticList diagnostics)
+ throws ClientProtocolException, IOException {
CloseableHttpClient client = HttpClientBuilder.create()
.setRedirectStrategy(new RedirectStrategy()).build();
CloseableHttpResponse response = client.execute(httpRequest);
@@ -228,6 +285,11 @@
logger.warn("Error response code: " + statusCode);
parseError(response);
}
+ else if (!freeAccess) {
+ diagnostics.addDiagnostic(SRUConstants.SRU_AUTHENTICATION_ERROR,
+ "The results could not be shown due to lack of authentication.",
+ "Authentication required.");
+ }
return response;
}
@@ -299,7 +361,7 @@
*/
private HttpGet createSearchRequest (String query,
QueryLanguage queryLanguage, String version, int startRecord,
- int maximumRecords, String[] corpora)
+ int maximumRecords, String corpusQuery, boolean freeAccess)
throws URISyntaxException, IOException, SRUException {
if (maximumRecords <= 0) {
@@ -310,43 +372,12 @@
defaultMaxRecords);
maximumRecords = defaultMaxRecords;
}
-
- String corpusQuery = "";
- boolean freeAccess = true;
- if (corpora != null && corpora.length > 0) {
- for (int i = 0; i < corpora.length; i++) {
- String pid = corpora[i];
- String cq = virtualCorpusQueries.get(pid);
-
- if (cq == null) {
- retrieveResources();
- cq = virtualCorpusQueries.get(pid);
- if (cq == null) {
- throw new SRUException(
- SRUConstants.SRU_GENERAL_SYSTEM_ERROR,
- "Virtual corpus with pid: " + pid
- + " is not found.");
- }
- }
- cq = URLDecoder.decode(cq, "utf-8");
- if (i == 0) {
- corpusQuery = cq;
- }
- else {
- corpusQuery += " & " + cq;
- }
-
- if (!virtualCorpusAccesses.get(pid)) {
- freeAccess = false;
- }
- }
- }
URI uri = createSearchUri(query, queryLanguage, version, startRecord,
maximumRecords, corpusQuery, freeAccess);
// logger.info("Query URI: " + uri.toString());
- System.out.println(uri.toString());
+// System.out.println(uri.toString());
HttpGet request = new HttpGet(uri);
return request;
}
@@ -439,7 +470,7 @@
CloseableHttpResponse response = null;
try {
- response = sendRequest(httpRequest);
+ response = sendRequest(httpRequest, true, null);
BufferedInputStream jsonStream =
new BufferedInputStream(response.getEntity().getContent());
diff --git a/src/main/java/de/ids_mannheim/korap/sru/KorapEndpointDescription.java b/src/main/java/de/ids_mannheim/korap/sru/KorapEndpointDescription.java
index c79d9d2..e81e352 100644
--- a/src/main/java/de/ids_mannheim/korap/sru/KorapEndpointDescription.java
+++ b/src/main/java/de/ids_mannheim/korap/sru/KorapEndpointDescription.java
@@ -128,15 +128,24 @@
}
for (KorapResource r : resources) {
- Map<String, String> description = null;
- String rDesc = r.getDescription();
- if (rDesc != null && !rDesc.isEmpty()) {
- description = new HashMap<String, String>();
- description.put("en", r.getDescription());
+ Map<String, String> descriptionMap = null;
+ String desc = r.getDescription();
+ if (desc != null && !desc.isEmpty()) {
+ descriptionMap = new HashMap<String, String>();
+ descriptionMap.put("en", desc);
}
+ // EM: this requires institution language in Kustvakt,
+ // omitted for now as it is optional
+ Map<String, String> institutionMap = null;
+// String inst = r.getInstitution();
+// if (inst!= null && !inst.isEmpty()) {
+// institutionMap = new HashMap<String, String>();
+// institutionMap.put("en", inst);
+// }
+
ResourceInfo ri = new ResourceInfo(r.getResourceId(), r.getTitles(),
- description, null, r.getLandingPage(),
+ descriptionMap, institutionMap, r.getLandingPage(),
Arrays.asList(r.getLanguages()),
AvailabilityRestriction.NONE, dataviews,
this.getSupportedLayers(), null);
diff --git a/src/main/java/de/ids_mannheim/korap/sru/KorapSRU.java b/src/main/java/de/ids_mannheim/korap/sru/KorapSRU.java
index af6db64..75906be 100644
--- a/src/main/java/de/ids_mannheim/korap/sru/KorapSRU.java
+++ b/src/main/java/de/ids_mannheim/korap/sru/KorapSRU.java
@@ -102,7 +102,7 @@
// logger.info("korapsru query: " + queryStr);
KorapResult korapResult = sendQuery(queryStr, request, version,
- queryLanguage);
+ queryLanguage, diagnostics);
checkKorapResultError(korapResult, queryLanguage,
isRewitesAllowed(request), diagnostics);
// logger.info("Number of records: "+korapResult.getTotalResults());
@@ -190,12 +190,13 @@
}
private KorapResult sendQuery(String queryStr, SRURequest request,
- String version, QueryLanguage queryLanguage) throws SRUException {
+ String version, QueryLanguage queryLanguage, SRUDiagnosticList
+ diagnostics) throws SRUException {
try {
return korapClient.query(queryStr, queryLanguage, version,
request.getStartRecord(), request.getMaximumRecords(),
- getCorporaList(request));
+ getCorporaList(request), diagnostics);
}
catch (HttpResponseException e) {
logger.warn("HttpResponseException: " + e.getStatusCode() + " "
diff --git a/src/test/java/de/ids_mannheim/korap/test/KorapClientTest.java b/src/test/java/de/ids_mannheim/korap/test/KorapClientTest.java
index 94e19e9..922526f 100644
--- a/src/test/java/de/ids_mannheim/korap/test/KorapClientTest.java
+++ b/src/test/java/de/ids_mannheim/korap/test/KorapClientTest.java
@@ -14,6 +14,8 @@
import de.ids_mannheim.korap.sru.KorapResource;
import de.ids_mannheim.korap.sru.KorapResult;
import de.ids_mannheim.korap.sru.QueryLanguage;
+import eu.clarin.sru.server.SRUDiagnostic;
+import eu.clarin.sru.server.SRUDiagnosticList;
import eu.clarin.sru.server.SRUException;
/**
@@ -38,7 +40,7 @@
createExpectationForSearch("der", "cql", "1.2", "50",
"search-der.jsonld");
- result = c.query("der", QueryLanguage.CQL, "1.2", 51, 1, null);
+ result = c.query("der", QueryLanguage.CQL, "1.2", 51, 1, null, null);
assertEquals(1, result.getMatchSize());
assertEquals(1858, result.getTotalResults());
@@ -63,7 +65,7 @@
"/corpus/GOE/AGF/00000/p7744-7745/matchInfo");
result = c.query("(\"blaue\"|\"grüne\")", QueryLanguage.FCSQL, "2.0", 1,
- 1, null);
+ 1, null,null);
assertEquals(1, result.getMatchSize());
assertEquals(55, result.getTotalResults());
@@ -112,7 +114,7 @@
throws HttpResponseException, Exception {
createExpectationForRetrieveResource();
KorapResource[] resources = c.retrieveResources();
- assertEquals(3, resources.length);
+ assertEquals(5, resources.length);
assertEquals("http://hdl.handle.net/10932/00-03B6-558F-4E10-6201-1",
resources[0].getResourceId());
assertEquals("http://hdl.handle.net/10932/00-03B6-558F-5EA0-6301-B",
@@ -127,9 +129,21 @@
createExpectationForSearch("\"Freizeit\"", "fcsql", "2.0", "0",
"textType = /.*[Rr]oman/", true,
"search-public-metadata.jsonld");
+
+ SRUDiagnosticList diagnostics = new SRUDiagnosticList() {
+
+ @Override
+ public void addDiagnostic (String uri, String details,
+ String message) {
+ System.out.println("Diagnostics: " + uri + "; Details: "
+ + details + "; Message: "+message);
+
+ }
+ };
+
KorapResult result = c.query("\"Freizeit\"", QueryLanguage.FCSQL, "2.0",
- 1, 1, new String[] { "Romane" });
+ 1, 1, new String[] { "Romane" }, diagnostics);
assertEquals(702, result.getTotalResults());
- assertEquals(0, result.getMatchSize());
+// assertEquals(0, result.getMatchSize());
}
}