Test simple text also without padding
Change-Id: I1b720659b38aabc613ef06b81b259a18c6eab142
diff --git a/src/test/java/org/ids_mannheim/WorkerTest.java b/src/test/java/org/ids_mannheim/WorkerTest.java
index 5e0623f..d18460a 100644
--- a/src/test/java/org/ids_mannheim/WorkerTest.java
+++ b/src/test/java/org/ids_mannheim/WorkerTest.java
@@ -225,7 +225,7 @@
}
@Test
- void paddingWorks() throws IOException {
+ void paddingWorksWithSimpleFile() throws IOException {
File tempFile = File.createTempFile("simple", ".conllu");
tempFile.deleteOnExit();
try (FileOutputStream out = new FileOutputStream(tempFile)) {
@@ -233,43 +233,46 @@
.getResourceAsStream("simple.conllu")), out);
}
- for (boolean with_lemma_and_pos : new boolean[]{false, true}) {
- for (int n = 1; n <= 3; n++) {
- ArrayList<String> fnames = new ArrayList<>();
- fnames.add(tempFile.getAbsolutePath());
+ for (boolean with_padding : new boolean[]{false, true}) {
+ for (boolean with_lemma_and_pos : new boolean[]{false, true}) {
+ for (int n = 1; n <= 3; n++) {
+ ArrayList<String> fnames = new ArrayList<>();
+ fnames.add(tempFile.getAbsolutePath());
- File tempFreqFile = File.createTempFile("simple", ".freq");
- tempFreqFile.deleteOnExit();
- try (FileOutputStream out = new FileOutputStream(tempFreqFile)) {
- IOUtils.copy(Objects.requireNonNull(Thread.currentThread().getContextClassLoader()
- .getResourceAsStream("simple_" + n + (with_lemma_and_pos ? "lp" : "") + "gram_padded.freq")), out);
+ File tempFreqFile = File.createTempFile("simple", ".freq");
+ tempFreqFile.deleteOnExit();
+ String goldFileName = "simple_" + n + (with_lemma_and_pos ? "lp" : "") + "gram" + (with_padding? "_padded" : "") + ".freq";
+ try (FileOutputStream out = new FileOutputStream(tempFreqFile)) {
+ IOUtils.copy(Objects.requireNonNull(Thread.currentThread().getContextClassLoader()
+ .getResourceAsStream(goldFileName)), out);
+ }
+
+ Path path = FileSystems.getDefault().getPath(tempFreqFile.getAbsolutePath());
+ Map<String, Integer> gold = Files.lines(path)
+ .filter(s -> s.matches(splitFreqlistRegex))
+ .collect(Collectors.toMap(k -> k.split(splitFreqlistRegex)[0], v -> Integer.parseInt(v.split(splitFreqlistRegex)[1])));
+
+ map = new ConcurrentHashMap<>();
+ LinkedBlockingQueue<Integer> queue = new LinkedBlockingQueue<>(2);
+ worker = new Worker(
+ queue,
+ fnames,
+ n,
+ 1,
+ 1,
+ map,
+ with_lemma_and_pos,
+ false,
+ new WorkerNodePool(""),
+ new Progressbar(tempFile.length()),
+ Logger.getLogger(TotalNGrams.class.getSimpleName()),
+ with_padding);
+
+ queue.add(0);
+ queue.add(-1);
+ worker.run();
+ gold.forEach((key, value) -> assertEquals(value, map.get(key).intValue()));
}
-
- Path path = FileSystems.getDefault().getPath(tempFreqFile.getAbsolutePath());
- Map<String, Integer> gold = Files.lines(path)
- .filter(s -> s.matches(splitFreqlistRegex))
- .collect(Collectors.toMap(k -> k.split(splitFreqlistRegex)[0], v -> Integer.parseInt(v.split(splitFreqlistRegex)[1])));
-
- map = new ConcurrentHashMap<>();
- LinkedBlockingQueue<Integer> queue = new LinkedBlockingQueue<>(2);
- worker = new Worker(
- queue,
- fnames,
- n,
- 1,
- 1,
- map,
- with_lemma_and_pos,
- false,
- new WorkerNodePool(""),
- new Progressbar(tempFile.length()),
- Logger.getLogger(TotalNGrams.class.getSimpleName()),
- true);
-
- queue.add(0);
- queue.add(-1);
- worker.run();
- gold.forEach((key, value) -> assertEquals(value, map.get(key).intValue()));
}
}
}