Serve a temporary file after event relocation
Change-Id: I7b87d8e73dc62db8a74d13a31c78fe783365d12b
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/ExWSConf.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/ExWSConf.java
index b4523d2..10c9030 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/ExWSConf.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/ExWSConf.java
@@ -1,6 +1,6 @@
/**
*
- * @author helge
+ * @author helge, ndiewald
*
* Class to define the constants of the export web service,
* like the maximum hits to be exported
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Exporter.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Exporter.java
index c82b35b..122d03d 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Exporter.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Exporter.java
@@ -12,6 +12,7 @@
// Implemented by MatchAggregator
public boolean init (String s) throws IOException;
+ public Exporter finish () throws IOException;
public void setMeta(JsonNode n);
public void setQuery(JsonNode n);
public void setCollection(JsonNode n);
@@ -21,6 +22,7 @@
public boolean appendMatches (String s) throws IOException;
public String getFileName ();
public void setFileName (String s);
+ public void setFile (String id);
public String getQueryString ();
public void setQueryString (String s);
public String getCorpusQueryString ();
@@ -31,8 +33,8 @@
public boolean hasTimeExceeded ();
public void setMaxResults (int m);
public void setSse (EventOutput sse);
-
- // Implemented by Exporter
+ public void forceFile ();
+ public String getExportID ();
public ResponseBuilder serve();
// Needs to be overwritten
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/MatchAggregator.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/MatchAggregator.java
index d3de004..2346d43 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/MatchAggregator.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/MatchAggregator.java
@@ -9,9 +9,9 @@
import java.util.Collection;
import java.util.ArrayList;
-
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.Properties;
import com.fasterxml.jackson.core.JsonFactory;
import com.fasterxml.jackson.core.JsonParser;
@@ -22,6 +22,7 @@
import javax.ws.rs.core.Response;
import javax.ws.rs.core.Response.ResponseBuilder;
+import javax.ws.rs.core.Response.Status;
import org.glassfish.jersey.media.sse.EventOutput;
import org.glassfish.jersey.media.sse.OutboundEvent;
@@ -35,6 +36,8 @@
public class MatchAggregator {
+ private final Properties prop = ExWSConf.properties(null);
+
private ObjectMapper mapper = new ObjectMapper();
private LinkedList<JsonNode> matches;
@@ -49,7 +52,7 @@
private int totalResults = -1;
private int maxResults = -1;
private int fetchedResults = 0;
-
+
private EventOutput evOut;
public String getMimeType() {
@@ -145,6 +148,22 @@
return this.collection;
};
+ public String getExportID () {
+ if (this.file == null)
+ return "";
+ return this.file.getName();
+ };
+
+ /**
+ * Set the file based on the export ID
+ */
+ public void setFile (String exportID) {
+ this.file = new File(
+ this.getFileDirectory(),
+ exportID
+ );
+ }
+
public void writeHeader (Writer w) throws IOException { };
public void writeFooter (Writer w) throws IOException { };
public void addMatch (JsonNode n, Writer w) throws IOException { };
@@ -152,6 +171,30 @@
public void setSse (EventOutput eventOutput) {
this.evOut = eventOutput;
};
+
+
+ private File getFileDirectory () {
+
+ String fileDir = prop.getProperty(
+ "conf.file_dir",
+ System.getProperty("java.io.tmpdir")
+ );
+
+ File dir = new File(fileDir);
+
+ // Create directory if not yet existing
+ if (!dir.exists()) {
+ dir.mkdir();
+ }
+
+ else if (!dir.canWrite()) {
+ fileDir = System.getProperty("java.io.tmpdir");
+ System.err.println("Unable to write to directory");
+ System.err.println("Fallback to " + fileDir);
+ dir = new File(fileDir);
+ };
+ return dir;
+ };
// Send the progress
private void sendProgress () {
@@ -176,6 +219,50 @@
};
/**
+ * Force creation of a file, even when only a few
+ * matches are requested.
+ */
+ public void forceFile () {
+
+ // Open file if not already opened
+ if (this.file == null) {
+
+ try {
+
+ File dir = getFileDirectory();
+
+ // Create temporary file
+ this.file = File.createTempFile(
+ "idsexp-", "." + this.getSuffix(),
+ dir
+ );
+
+ // better delete after it is not needed anymore
+ // this.file.deleteOnExit();
+
+ String s = null;
+
+ if (writer != null)
+ s = writer.toString();
+
+ // Establish writer
+ writer = new BufferedWriter(new FileWriter(this.file, true));
+
+ // Add in memory string
+ if (s != null)
+ writer.write(s);
+
+ }
+ catch (IOException e) {
+
+ // Will rely on in-memory data
+ return;
+ };
+ };
+ };
+
+
+ /**
* Create new match aggregator and parse initial Json
* file to get header information and initial matches.
*/
@@ -223,23 +310,8 @@
*/
public boolean appendMatches (String resp) throws IOException {
- // Open a temp file if not already opened
- if (this.file == null) {
-
- // Create temporary file
- this.file = File.createTempFile("idsexppl-", ".tmpJson");
-
- // better delete after it is not needed anymore
- this.file.deleteOnExit();
-
- String s = writer.toString();
-
- // Establish writer
- writer = new BufferedWriter(new FileWriter(this.file, true));
-
- // Add in memory string
- writer.write(s);
- };
+ // Demand creation of a file
+ this.forceFile();
JsonParser parser = mapper.getFactory().createParser(resp);
JsonNode actualObj = mapper.readTree(parser);
@@ -252,43 +324,47 @@
);
};
+ /**
+ * Finalize the export stream.
+ */
+ public Exporter finish() throws IOException {
+ this.writeFooter(this.writer);
+ this.writer.close();
+ return (Exporter) this;
+ };
+
/**
* Serve response entity, either as a string or as a file.
*/
public ResponseBuilder serve () {
- try {
- ResponseBuilder rb;
- this.writeFooter(this.writer);
- this.writer.close();
+ ResponseBuilder rb;
+ if (this.file == null) {
- if (this.file == null) {
- rb = Response.ok(writer.toString());
- }
- else {
- rb = Response.ok(this.file);
- };
-
- return rb
- .type(this.getMimeType())
- .header(
- "Content-Disposition",
- "attachment; filename=" +
- this.getFileName() +
- '.' +
- this.getSuffix()
- );
+ // Serve stream
+ rb = Response.ok(writer.toString());
}
+ else if (this.file.exists()) {
- // Catch error
- catch (IOException io) {
+ // Serve file
+ rb = Response.ok(this.file);
+ }
+ else {
+ // File doesn't exist
+ return Response.status(Status.NOT_FOUND);
};
- // TODO:
- // Return exporter error
- return Response.status(500).entity("error");
+ return rb
+ .type(this.getMimeType())
+ .header(
+ "Content-Disposition",
+ "attachment; filename=" +
+ this.getFileName() +
+ '.' +
+ this.getSuffix()
+ );
};
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Service.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Service.java
index 9be9d9e..df3e53c 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Service.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Service.java
@@ -23,6 +23,7 @@
import javax.ws.rs.WebApplicationException;
import javax.ws.rs.FormParam;
import javax.ws.rs.QueryParam;
+import javax.ws.rs.PathParam;
import javax.ws.rs.POST;
import javax.ws.rs.GET;
import javax.ws.rs.Path;
@@ -39,6 +40,8 @@
import javax.ws.rs.core.Response.ResponseBuilder;
import javax.ws.rs.core.Response.Status;
+import static org.apache.commons.io.FilenameUtils.getExtension;
+
import org.glassfish.jersey.media.sse.EventOutput;
import org.glassfish.jersey.media.sse.OutboundEvent;
import org.glassfish.jersey.media.sse.SseFeature;
@@ -55,9 +58,9 @@
/**
* TODO:
* - Delete the temp file of the export at the end
+ * of the serving.
* - Do not expect all meta data per match.
* - Abort processing when eventsource is closed.
- * - Add progress mechanism.
* - Upgrade default pageSize to 50.
* - Add loading marker.
* - Add hitc to form.
@@ -68,7 +71,7 @@
@Path("/")
public class Service {
- Properties prop = ExWSConf.properties(null);
+ private Properties prop = ExWSConf.properties(null);
private final ClassLoader cl = Thread.currentThread().getContextClassLoader();
@@ -96,7 +99,7 @@
// Private method to run the export,
// either static or streaming
- private Exporter export (String fname,
+ private Exporter export(String fname,
String format,
String q,
String cq,
@@ -197,16 +200,7 @@
);
}
- Exporter exp;
-
- // Choose the correct exporter
- if (format.equals("json"))
- exp = new JsonExporter();
- else if (format.equals("csv"))
- exp = new CsvExporter();
- else
- exp = new RtfExporter();
-
+ Exporter exp = getExporter(format);
exp.setMaxResults(maxResults);
exp.setQueryString(q);
exp.setCorpusQueryString(cq);
@@ -218,20 +212,15 @@
};
// set progress mechanism, if required
- if (eventOutput != null)
+ if (eventOutput != null) {
exp.setSse(eventOutput);
+ exp.forceFile();
+ };
- // TODO:
- // The following could be subsumed in the MatchAggregator
- // as a "run()" routine.
-
-
// Initialize exporter (with meta data and first matches)
try {
exp.init(resp);
-
} catch (Exception e) {
-
throw new WebApplicationException(
responseForm(
Status.INTERNAL_SERVER_ERROR,
@@ -257,6 +246,16 @@
// If only one page should be exported there is no need
// for a temporary export file
if (cutoff) {
+ try {
+ exp.finish();
+ } catch (Exception e) {
+ throw new WebApplicationException(
+ responseForm(
+ Status.INTERNAL_SERVER_ERROR,
+ e.getMessage()
+ )
+ );
+ };
return exp;
};
@@ -280,6 +279,9 @@
if (!exp.appendMatches(resp))
break;
}
+
+ exp.finish();
+
} catch (Exception e) {
throw new WebApplicationException(
responseForm(
@@ -378,10 +380,11 @@
hitc,
eventOutput
);
+
if (eventOutput.isClosed())
return;
eventBuilder.name("Relocate");
- eventBuilder.data("...");
+ eventBuilder.data(exp.getExportID());
eventOutput.write(eventBuilder.build());
} catch (Exception e) {
try {
@@ -415,6 +418,32 @@
.build();
};
+
+ /**
+ * This is the relocation target to which the event
+ * stream points to.
+ */
+ @GET
+ @Path("export/{file}")
+ @Produces(MediaType.APPLICATION_OCTET_STREAM)
+ public Response fileExport(
+ @PathParam("file") String fileStr,
+ @QueryParam("fname") String fname
+ ) {
+
+ String format = getExtension(fileStr);
+
+ // Get exporter object
+ Exporter exp = getExporter(format);
+ if (fname != null) {
+ exp.setFileName(fname);
+ };
+ exp.setFile(fileStr);
+
+ // Return without init
+ return exp.serve().build();
+ };
+
@GET
@Path("export")
@@ -432,7 +461,18 @@
.ok(exportJsStr, "application/javascript")
.build();
};
-
+
+ // Get exporter by format
+ private Exporter getExporter (String format) {
+ // Choose the correct exporter
+ if (format.equals("json"))
+ return new JsonExporter();
+ else if (format.equals("csv"))
+ return new CsvExporter();
+
+ return new RtfExporter();
+ };
+
// Decorate request with auth headers
private Invocation.Builder authBuilder (Invocation.Builder reqBuilder,
diff --git a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Util.java b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Util.java
index 0ca1a94..cc024e9 100644
--- a/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Util.java
+++ b/plugin/src/main/java/de/ids_mannheim/korap/plkexport/Util.java
@@ -13,7 +13,7 @@
.replaceFirst("^-+","")
.replaceFirst("-+$","")
;
- }
+ };
public static String streamToString (InputStream in) {
StringBuilder sb = new StringBuilder();
@@ -30,5 +30,5 @@
}
return sb.toString();
- }
-}
+ };
+};