totalngrams: improve logging
* use <output_file>.log by default
* one line per log
* still in default locale :)
diff --git a/src/main/java/org/ids_mannheim/TotalNGram.java b/src/main/java/org/ids_mannheim/TotalNGram.java
index 6e0add0..5de0f84 100644
--- a/src/main/java/org/ids_mannheim/TotalNGram.java
+++ b/src/main/java/org/ids_mannheim/TotalNGram.java
@@ -10,7 +10,6 @@
import java.nio.file.FileAlreadyExistsException;
import java.nio.file.Files;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.Locale;
import java.util.concurrent.*;
import java.util.logging.FileHandler;
@@ -25,7 +24,8 @@
public class TotalNGram implements Callable<Integer> {
private static final int MAX_THREADS = Runtime.getRuntime().availableProcessors() * 2 / 3;
- static private final Logger logger;
+ private static final String DEFAULT_LOGFILE = "totalngram.log";
+ static public final Logger logger;
static {
String path = TotalNGram.class.getClassLoader()
@@ -38,7 +38,7 @@
@CommandLine.Parameters(arity = "1..*", description = "input files")
private final ArrayList<String> inputFiles = new ArrayList<>();
@CommandLine.Option(names = {"-L",
- "--log-file"}, defaultValue = "totalngram.log", description = "log file name (default: ${DEFAULT-VALUE})")
+ "--log-file"}, defaultValue = DEFAULT_LOGFILE, description = "log file name (default: ${DEFAULT-VALUE})")
String logFileName;
@SuppressWarnings("CanBeFinal")
@@ -47,7 +47,7 @@
String output_fillename = "-";
@SuppressWarnings("CanBeFinal")
- @CommandLine.Option(names = {"--force"}, description = "Force overwrrite (default: ${DEFAULT-VALUE})")
+ @CommandLine.Option(names = {"--force"}, description = "Force overwrite (default: ${DEFAULT-VALUE})")
boolean force_overwrite = false;
@SuppressWarnings("CanBeFinal")
@@ -78,19 +78,27 @@
System.exit(new CommandLine(new TotalNGram()).execute(args));
}
- @Override
- public Integer call() throws Exception {
+ private void initLogging() {
+ if (logFileName.equals(DEFAULT_LOGFILE) && !output_fillename.equals("-")) {
+ logFileName = Utils.newExtension(output_fillename, "log");
+ }
try {
FileHandler fileHandler = new FileHandler(logFileName);
+ System.setProperty("java.util.logging.SimpleFormatter.format",
+ "[%1$tF %1$tT] [%4$-7s] %5$s %n");
fileHandler.setFormatter(new SimpleFormatter());
//java.util.logging.SimpleFormatter.format="%1$tb %1$td, %1$tY %1$tl:%1$tM:%1$tS %1$Tp %2$s%n%4$s: %5$s%n";
java.util.Locale.setDefault(Locale.Category.FORMAT, Locale.ROOT);
logger.addHandler(fileHandler);
-
} catch (Exception e) {
+ System.err.println("Cannot write logfile " + logFileName + ": " + e);
+ System.exit(-1);
}
+ }
- PrintStream output_stream;
+ @Override
+ public Integer call() throws Exception {
+ initLogging();
final PrintStream output_stream;
if ((output_fillename == null) || output_fillename.equals("-")) {
output_stream = System.out;
@@ -126,6 +134,7 @@
max_threads = workerNodePool.size;
}
int threads = Math.min(max_threads, inputFiles.size());
+ logger.info("Using " + threads + " threads");
IntStream.range(0, threads).forEach(unused -> es.execute(new Worker(queue, inputFiles, ngram_size, FOLDS, map, workerNodePool, etaPrinter, logger)));
queue.addAll(IntStream.range(0, inputFiles.size()).boxed().collect(Collectors.toList()));
IntStream.range(0, threads).forEach(unused -> {
diff --git a/src/main/java/org/ids_mannheim/Utils.java b/src/main/java/org/ids_mannheim/Utils.java
new file mode 100644
index 0000000..7d60dc9
--- /dev/null
+++ b/src/main/java/org/ids_mannheim/Utils.java
@@ -0,0 +1,10 @@
+package org.ids_mannheim;
+
+public class Utils {
+ public static String newExtension(String original, String newExtension) {
+ if (original.contains("."))
+ return original.replaceFirst("(\\.[^.]*|\\.[^.]+\\.gz)$", "."+newExtension);
+ else
+ return original+".log";
+ }
+}
diff --git a/src/main/resources/logging.properties b/src/main/resources/logging.properties
index c793dcc..ce9250c 100644
--- a/src/main/resources/logging.properties
+++ b/src/main/resources/logging.properties
@@ -1,9 +1,13 @@
handlers= java.util.logging.FileHandler
.level= INFO
+java.util.logging.SimpleFormatter.format=[%1$tF %1$tT] [%4$-7s] %5$s %n
java.util.logging.FileHandler.level = INFO
java.util.logging.FileHandler.pattern = totalngrams.log
java.util.logging.FileHandler.limit = 50000
java.util.logging.FileHandler.count = 1
java.util.logging.FileHandler.formatter = java.util.logging.SimpleFormatter
# java.util.logging.SimpleFormatter.format=%4$s: %5$s [%1$tc]%n
-java.util.logging.SimpleFormatter.format="%1$tb %1$td, %1$tY %1$tH:%1$tM:%1$tS %1$Tp %2$s%n%4$s: %5$s%n"
+# java.util.logging.SimpleFormatter.format="%1$tb %1$td, %1$tY %1$tH:%1$tM:%1$tS %1$Tp %2$s%n%4$s: %5$s%n"
+
+java.util.logging.ConsoleHandler.level = WARNING
+java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter