Serve a temporary file after event relocation
Change-Id: I7b87d8e73dc62db8a74d13a31c78fe783365d12b
diff --git a/plugin/pom.xml b/plugin/pom.xml
index d5a86f3..c93342b 100644
--- a/plugin/pom.xml
+++ b/plugin/pom.xml
@@ -111,6 +111,12 @@
<version>${jersey.version}</version>
</dependency>
+ <dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <version>2.8.0</version>
+ </dependency>
+
</dependencies>
<build>
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/ExWSConf.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/ExWSConf.java
index b4523d2..10c9030 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/ExWSConf.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/ExWSConf.java
@@ -1,6 +1,6 @@
/**
*
- * @author helge
+ * @author helge, ndiewald
*
* Class to define the constants of the export web service,
* like the maximum hits to be exported
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Exporter.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Exporter.java
index c82b35b..122d03d 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Exporter.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Exporter.java
@@ -12,6 +12,7 @@
// Implemented by MatchAggregator
public boolean init (String s) throws IOException;
+ public Exporter finish () throws IOException;
public void setMeta(JsonNode n);
public void setQuery(JsonNode n);
public void setCollection(JsonNode n);
@@ -21,6 +22,7 @@
public boolean appendMatches (String s) throws IOException;
public String getFileName ();
public void setFileName (String s);
+ public void setFile (String id);
public String getQueryString ();
public void setQueryString (String s);
public String getCorpusQueryString ();
@@ -31,8 +33,8 @@
public boolean hasTimeExceeded ();
public void setMaxResults (int m);
public void setSse (EventOutput sse);
-
- // Implemented by Exporter
+ public void forceFile ();
+ public String getExportID ();
public ResponseBuilder serve();
// Needs to be overwritten
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/MatchAggregator.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/MatchAggregator.java
index d3de004..2346d43 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/MatchAggregator.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/MatchAggregator.java
@@ -9,9 +9,9 @@
import java.util.Collection;
import java.util.ArrayList;
-
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.Properties;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
@@ -22,6 +22,7 @@
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.ResponseBuilder;
+import javax.ws.rs.core.Response.Status;
import org.glassfish.jersey.media.sse.EventOutput;
import org.glassfish.jersey.media.sse.OutboundEvent;
@@ -35,6 +36,8 @@
public class MatchAggregator {
+ private final Properties prop = ExWSConf.properties(null);
+
private ObjectMapper mapper = new ObjectMapper();
private LinkedList<JsonNode> matches;
@@ -49,7 +52,7 @@
private int totalResults = -1;
private int maxResults = -1;
private int fetchedResults = 0;
-
+
private EventOutput evOut;
public String getMimeType() {
@@ -145,6 +148,22 @@
return this.collection;
};
+ public String getExportID () {
+ if (this.file == null)
+ return "";
+ return this.file.getName();
+ };
+
+ /**
+ * Set the file based on the export ID
+ */
+ public void setFile (String exportID) {
+ this.file = new File(
+ this.getFileDirectory(),
+ exportID
+ );
+ }
+
public void writeHeader (Writer w) throws IOException { };
public void writeFooter (Writer w) throws IOException { };
public void addMatch (JsonNode n, Writer w) throws IOException { };
@@ -152,6 +171,30 @@
public void setSse (EventOutput eventOutput) {
this.evOut = eventOutput;
};
+
+
+ private File getFileDirectory () {
+
+ String fileDir = prop.getProperty(
+ "conf.file_dir",
+ System.getProperty("java.io.tmpdir")
+ );
+
+ File dir = new File(fileDir);
+
+ // Create directory if not yet existing
+ if (!dir.exists()) {
+ dir.mkdir();
+ }
+
+ else if (!dir.canWrite()) {
+ fileDir = System.getProperty("java.io.tmpdir");
+ System.err.println("Unable to write to directory");
+ System.err.println("Fallback to " + fileDir);
+ dir = new File(fileDir);
+ };
+ return dir;
+ };
// Send the progress
private void sendProgress () {
@@ -176,6 +219,50 @@
};
/**
+ * Force creation of a file, even when only a few
+ * matches are requested.
+ */
+ public void forceFile () {
+
+ // Open file if not already opened
+ if (this.file == null) {
+
+ try {
+
+ File dir = getFileDirectory();
+
+ // Create temporary file
+ this.file = File.createTempFile(
+ "idsexp-", "." + this.getSuffix(),
+ dir
+ );
+
+ // better delete after it is not needed anymore
+ // this.file.deleteOnExit();
+
+ String s = null;
+
+ if (writer != null)
+ s = writer.toString();
+
+ // Establish writer
+ writer = new BufferedWriter(new FileWriter(this.file, true));
+
+ // Add in memory string
+ if (s != null)
+ writer.write(s);
+
+ }
+ catch (IOException e) {
+
+ // Will rely on in-memory data
+ return;
+ };
+ };
+ };
+
+
+ /**
* Create new match aggregator and parse initial Json
* file to get header information and initial matches.
*/
@@ -223,23 +310,8 @@
*/
public boolean appendMatches (String resp) throws IOException {
- // Open a temp file if not already opened
- if (this.file == null) {
-
- // Create temporary file
- this.file = File.createTempFile("idsexppl-", ".tmpJson");
-
- // better delete after it is not needed anymore
- this.file.deleteOnExit();
-
- String s = writer.toString();
-
- // Establish writer
- writer = new BufferedWriter(new FileWriter(this.file, true));
-
- // Add in memory string
- writer.write(s);
- };
+ // Demand creation of a file
+ this.forceFile();
JsonParser parser = mapper.getFactory().createParser(resp);
JsonNode actualObj = mapper.readTree(parser);
@@ -252,43 +324,47 @@
);
};
+ /**
+ * Finalize the export stream.
+ */
+ public Exporter finish() throws IOException {
+ this.writeFooter(this.writer);
+ this.writer.close();
+ return (Exporter) this;
+ };
+
/**
* Serve response entity, either as a string or as a file.
*/
public ResponseBuilder serve () {
- try {
- ResponseBuilder rb;
- this.writeFooter(this.writer);
- this.writer.close();
+ ResponseBuilder rb;
+ if (this.file == null) {
- if (this.file == null) {
- rb = Response.ok(writer.toString());
- }
- else {
- rb = Response.ok(this.file);
- };
-
- return rb
- .type(this.getMimeType())
- .header(
- "Content-Disposition",
- "attachment; filename=" +
- this.getFileName() +
- '.' +
- this.getSuffix()
- );
+ // Serve stream
+ rb = Response.ok(writer.toString());
}
+ else if (this.file.exists()) {
- // Catch error
- catch (IOException io) {
+ // Serve file
+ rb = Response.ok(this.file);
+ }
+ else {
+ // File doesn't exist
+ return Response.status(Status.NOT_FOUND);
};
- // TODO:
- // Return exporter error
- return Response.status(500).entity("error");
+ return rb
+ .type(this.getMimeType())
+ .header(
+ "Content-Disposition",
+ "attachment; filename=" +
+ this.getFileName() +
+ '.' +
+ this.getSuffix()
+ );
};
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Service.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Service.java
index 9be9d9e..df3e53c 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Service.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Service.java
@@ -23,6 +23,7 @@
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.FormParam;
import javax.ws.rs.QueryParam;
+import javax.ws.rs.PathParam;
import javax.ws.rs.POST;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
@@ -39,6 +40,8 @@
import javax.ws.rs.core.Response.ResponseBuilder;
import javax.ws.rs.core.Response.Status;
+import static org.apache.commons.io.FilenameUtils.getExtension;
+
import org.glassfish.jersey.media.sse.EventOutput;
import org.glassfish.jersey.media.sse.OutboundEvent;
import org.glassfish.jersey.media.sse.SseFeature;
@@ -55,9 +58,9 @@
/**
* TODO:
* - Delete the temp file of the export at the end
+ * of the serving.
* - Do not expect all meta data per match.
* - Abort processing when eventsource is closed.
- * - Add progress mechanism.
* - Upgrade default pageSize to 50.
* - Add loading marker.
* - Add hitc to form.
@@ -68,7 +71,7 @@
@Path("/")
public class Service {
- Properties prop = ExWSConf.properties(null);
+ private Properties prop = ExWSConf.properties(null);
private final ClassLoader cl = Thread.currentThread().getContextClassLoader();
@@ -96,7 +99,7 @@
// Private method to run the export,
// either static or streaming
- private Exporter export (String fname,
+ private Exporter export(String fname,
String format,
String q,
String cq,
@@ -197,16 +200,7 @@
);
}
- Exporter exp;
-
- // Choose the correct exporter
- if (format.equals("json"))
- exp = new JsonExporter();
- else if (format.equals("csv"))
- exp = new CsvExporter();
- else
- exp = new RtfExporter();
-
+ Exporter exp = getExporter(format);
exp.setMaxResults(maxResults);
exp.setQueryString(q);
exp.setCorpusQueryString(cq);
@@ -218,20 +212,15 @@
};
// set progress mechanism, if required
- if (eventOutput != null)
+ if (eventOutput != null) {
exp.setSse(eventOutput);
+ exp.forceFile();
+ };
- // TODO:
- // The following could be subsumed in the MatchAggregator
- // as a "run()" routine.
-
-
// Initialize exporter (with meta data and first matches)
try {
exp.init(resp);
-
} catch (Exception e) {
-
throw new WebApplicationException(
responseForm(
Status.INTERNAL_SERVER_ERROR,
@@ -257,6 +246,16 @@
// If only one page should be exported there is no need
// for a temporary export file
if (cutoff) {
+ try {
+ exp.finish();
+ } catch (Exception e) {
+ throw new WebApplicationException(
+ responseForm(
+ Status.INTERNAL_SERVER_ERROR,
+ e.getMessage()
+ )
+ );
+ };
return exp;
};
@@ -280,6 +279,9 @@
if (!exp.appendMatches(resp))
break;
}
+
+ exp.finish();
+
} catch (Exception e) {
throw new WebApplicationException(
responseForm(
@@ -378,10 +380,11 @@
hitc,
eventOutput
);
+
if (eventOutput.isClosed())
return;
eventBuilder.name("Relocate");
- eventBuilder.data("...");
+ eventBuilder.data(exp.getExportID());
eventOutput.write(eventBuilder.build());
} catch (Exception e) {
try {
@@ -415,6 +418,32 @@
.build();
};
+
+ /**
+ * This is the relocation target to which the event
+ * stream points to.
+ */
+ @GET
+ @Path("export/{file}")
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ public Response fileExport(
+ @PathParam("file") String fileStr,
+ @QueryParam("fname") String fname
+ ) {
+
+ String format = getExtension(fileStr);
+
+ // Get exporter object
+ Exporter exp = getExporter(format);
+ if (fname != null) {
+ exp.setFileName(fname);
+ };
+ exp.setFile(fileStr);
+
+ // Return without init
+ return exp.serve().build();
+ };
+
@GET
@Path("export")
@@ -432,7 +461,18 @@
.ok(exportJsStr, "application/javascript")
.build();
};
-
+
+ // Get exporter by format
+ private Exporter getExporter (String format) {
+ // Choose the correct exporter
+ if (format.equals("json"))
+ return new JsonExporter();
+ else if (format.equals("csv"))
+ return new CsvExporter();
+
+ return new RtfExporter();
+ };
+
// Decorate request with auth headers
private Invocation.Builder authBuilder (Invocation.Builder reqBuilder,
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Util.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Util.java
index 0ca1a94..cc024e9 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Util.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Util.java
@@ -13,7 +13,7 @@
.replaceFirst("^-+","")
.replaceFirst("-+$","")
;
- }
+ };
public static String streamToString (InputStream in) {
StringBuilder sb = new StringBuilder();
@@ -30,5 +30,5 @@
}
return sb.toString();
- }
-}
+ };
+};
diff --git a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/CsvExporterTest.java b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/CsvExporterTest.java
index 528c63f..4cacacc 100644
--- a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/CsvExporterTest.java
+++ b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/CsvExporterTest.java
@@ -17,6 +17,7 @@
public void testInit () throws IOException {
CsvExporter csv = new CsvExporter();
csv.init("{\"query\":\"cool\"}");
+ csv.finish();
Response resp = csv.serve().build();
String x = (String) resp.getEntity();
@@ -45,6 +46,7 @@
"<span class=\\\"context-right\\\"> Snippet"+
"<span class=\\\"more\\\"></span></span>\"}"+
"]}");
+ csv.finish();
Response resp = csv.serve().build();
String x = (String) resp.getEntity();
diff --git a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/JsonExporterTest.java b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/JsonExporterTest.java
index 4bea7db..79a443d 100644
--- a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/JsonExporterTest.java
+++ b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/JsonExporterTest.java
@@ -20,7 +20,7 @@
public void testInit () throws IOException {
JsonExporter json = new JsonExporter();
json.init("{\"query\":\"cool\"}");
-
+ json.finish();
Response resp = json.serve().build();
String x = (String) resp.getEntity();
resp.close();
@@ -31,6 +31,7 @@
public void testInitFull () throws IOException {
JsonExporter json = new JsonExporter();
json.init("{\"meta\":\"ja\",\"collection\":\"hm\",\"query\":\"cool\",\"matches\":[\"first\",\"second\"]}");
+ json.finish();
Response resp = json.serve().build();
String x = (String) resp.getEntity();
@@ -43,6 +44,7 @@
JsonExporter json = new JsonExporter();
json.init("{\"meta\":\"ja\",\"collection\":\"hm\",\"query\":\"cool\",\"matches\":[\"first\",\"second\"]}");
json.appendMatches("{\"meta\":\"ja2\",\"collection\":\"hm2\",\"query\":\"cool2\",\"matches\":[\"third\",\"fourth\"]}");
+ json.finish();
Response resp = json.serve().build();
File x = (File) resp.getEntity();
diff --git a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/MatchAggregatorTest.java b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/MatchAggregatorTest.java
index 52a8f71..5acf32e 100644
--- a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/MatchAggregatorTest.java
+++ b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/MatchAggregatorTest.java
@@ -27,7 +27,7 @@
assertNull(m.getQuery());
assertNull(m.getCollection());
};
-
+
@Test
public void testSampleInit () throws IOException {
MatchAggregator m = new MatchAggregator();
@@ -118,4 +118,11 @@
assertEquals(m.getSource(),"/path");
};
+ @Test
+ public void testFileHandling () throws IOException {
+ MatchAggregator m = new MatchAggregator();
+ assertEquals(m.getExportID(), "");
+ m.forceFile();
+ assertTrue(m.getExportID().contains("idsexp-"));
+ };
};
diff --git a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/RtfExporterTest.java b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/RtfExporterTest.java
index a146845..311d479 100644
--- a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/RtfExporterTest.java
+++ b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/RtfExporterTest.java
@@ -17,6 +17,7 @@
public void testInit () throws IOException {
RtfExporter rtf = new RtfExporter();
rtf.init("{\"query\":\"cool\"}");
+ rtf.finish();
Response resp = rtf.serve().build();
String x = (String) resp.getEntity();
@@ -45,6 +46,7 @@
"<span class=\\\"context-right\\\"> Snippet"+
"<span class=\\\"more\\\"></span></span>\"}"+
"]}");
+ rtf.finish();
Response resp = rtf.serve().build();
String x = (String) resp.getEntity();
@@ -70,6 +72,7 @@
"\"textSigle\":\"RTF/G59/34284\","+
"\"snippet\":\"<span class=\\\"context-left\\\"></span><span class=\\\"match\\\"><mark>Und dafür musstest Du extra ne neue Socke erstellen? Wieso traust Du Dich nicht, mit Deinem Account aufzutreten? - -- ωωσσI - talk with me 09:17, 17. Dez. 2011 (CET) Der ist doch gesperrt. -- 09:21, 17. Dez. 2011 (CET) WWSS1, weil ich normalerweise mit IP schreibe und in dem Fall nicht möchte, dass</mark><span class=\\\"cutted\\\"></span></span><span class=\\\"context-right\\\"> meine IP öffentlich angezeigt wird. Über die IP kann man auf den Wohnort, den Provider und bei Aufenthalt am Arbeitsplatz auf den Arbeitgeber schließen, über Konto nicht. -- 09:24, 17. Dez. 2011 (CET) Bist Du denn nicht mehr selber Arbeitgeber? -- 09:31<span class=\\\"more\\\"></span></span>\"}"+
"]}");
+ rtf.finish();
Response resp = rtf.serve().build();
String x = (String) resp.getEntity();
diff --git a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/ServiceTest.java b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/ServiceTest.java
index 6c51f5e..6bbcc00 100644
--- a/plugin/src/test/java/de/ids_mannheim/korap/plkexport/ServiceTest.java
+++ b/plugin/src/test/java/de/ids_mannheim/korap/plkexport/ServiceTest.java
@@ -785,11 +785,14 @@
@Test
public void testExportWsProgressError () throws InterruptedException {
+
+ // TODO:
+ // Make this threadsafe
final LinkedList<String> events = new LinkedList<>();
- final int eventCount = 3;
+ // final int eventCount = 3;
// Expect messages:
- final CountDownLatch latch = new CountDownLatch(eventCount);
+ // final CountDownLatch latch = new CountDownLatch(eventCount);
// Create SSE client
Client client = ClientBuilder
@@ -804,13 +807,13 @@
EventListener listener = inboundEvent -> {
events.add(inboundEvent.getName() + ":" + inboundEvent.readData(String.class));
- latch.countDown();
+ // latch.countDown();
};
eventSource.register(listener);
eventSource.open();
- latch.await(1000, TimeUnit.SECONDS);
+ // latch.await(1000, TimeUnit.SECONDS);
Thread.sleep(2000);
// Check error
@@ -888,12 +891,52 @@
assertEquals(events.getFirst(), "Process:init");
assertEquals(events.get(1), "Progress:0");
assertEquals(events.get(2), "Progress:56");
- assertEquals(events.get(3), "Relocate:...");
+ assertTrue(events.get(3).startsWith("Relocate:"));
assertEquals(events.getLast(), "Process:done");
assertEquals(events.size(), 5);
eventSource.close();
+
+ // Now fetch the file!
+ String fileLoc = events.get(3).substring(9);
+
+ String filename = "ExampleHui";
+ Response response = target("/export/" + fileLoc).queryParam("fname", filename).request().get();
+
+ String str = response.readEntity(String.class);
+
+ assertEquals("HTTP Code",
+ Status.OK.getStatusCode(), response.getStatus());
+ assertTrue(
+ "Request: Results should not be displayed inline, but saved and displayed locally",
+ response.getHeaderString(HttpHeaders.CONTENT_DISPOSITION)
+ .contains("attachment"));
+
+ assertTrue("Request: Filename should be set correctly: ",
+ response.getHeaderString(HttpHeaders.CONTENT_DISPOSITION)
+ .contains("filename=" + filename + ".rtf"));
+
+ // An RTF document should be returned
+ assertEquals("Request RTF: Http Content-Type should be: ",
+ "application/rtf",
+ response.getHeaderString(HttpHeaders.CONTENT_TYPE));
+
+
+ assertTrue("Intro", str.contains("{\\rtf1\\ansi\\deff0"));
+ assertTrue("Outro", str.contains("{\\pard\\brdrb\\brdrs\\brdrw2\\brsp20\\par}"));
+ assertTrue("Content", str.contains("Benutzer Diskussion:Kriddl"));
};
+
+ @Test
+ public void testFileServingError () {
+ String fileLoc = "hjGHJghjgHJGhjghj";
+ Response response = target("/export/" + fileLoc).request().get();
+
+ assertEquals("HTTP Code",
+ Status.NOT_FOUND.getStatusCode(), response.getStatus());
+
+ };
+
@Test
public void testClientIP () {