commit 010903ea33da00c7c0856f85b34188f4c3f256ad
author Marc Kupietz <kupietz@ids-mannheim.de> Thu Jul 23 13:42:10 2020 +0200
committer Marc Kupietz <kupietz@ids-mannheim.de> Thu Jul 23 13:42:10 2020 +0200
tree 4fe674703cdd210c614b8a30e5e2b7c670f39c2b
parent e31c4e1770f2547e3ccbea0ee7adfe369d17029f

totalngrams: reuse slidingWindowQueue

src/main/java/org/ids_mannheim/SlidingWindowQueue.java

diff --git a/src/main/java/org/ids_mannheim/SlidingWindowQueue.java b/src/main/java/org/ids_mannheim/SlidingWindowQueue.java
index 4f07362..d54306c 100644
--- a/src/main/java/org/ids_mannheim/SlidingWindowQueue.java
+++ b/src/main/java/org/ids_mannheim/SlidingWindowQueue.java
@@ -1,13 +1,17 @@
 package org.ids_mannheim;
 
 import java.util.LinkedList;
-import java.util.function.Consumer;
 
 public class SlidingWindowQueue extends LinkedList {
     private final int maxSize;
-    private final Consumer<String> flush;
+    private final Increaser flush;
+    public int fold;
 
-    public SlidingWindowQueue(int size, Consumer<String> flush) {
+    interface Increaser {
+        void accept(String s, int fold);
+    }
+
+    public SlidingWindowQueue(int size, Increaser flush) {
         this.maxSize = size;
         this.flush = flush;
     }
@@ -15,9 +19,14 @@
     public boolean add(String k) {
         boolean r = super.add(k);
         if (size() == maxSize) {
-            this.flush.accept(String.join(" ", this));
+            this.flush.accept(String.join(" ", this), fold);
             remove(0);
         }
         return r;
     }
+
+    public void reset(int fold) {
+        this.clear();
+        this.fold = fold;
+    }
 }

src/main/java/org/ids_mannheim/Worker.java

diff --git a/src/main/java/org/ids_mannheim/Worker.java b/src/main/java/org/ids_mannheim/Worker.java
index 0a62f68..1578a98 100644
--- a/src/main/java/org/ids_mannheim/Worker.java
+++ b/src/main/java/org/ids_mannheim/Worker.java
@@ -44,6 +44,7 @@
         try {
             int index = queue.take();
             int retries = MAX_RETRIES;
+            SlidingWindowQueue slidingWindowQueue = new SlidingWindowQueue(ngram_size, (s, f) -> FoldedEntry.incr(map, s, f));
             while (index >= 0) {
                 String fname = fnames.get(index);
                 File current_file = new File(fname);
@@ -59,7 +60,6 @@
                 BufferedReader in = new BufferedReader(new InputStreamReader(p.getInputStream()));
                 String line;
                 int fold = -1;
-                SlidingWindowQueue slidingWindowQueue = null;
                 int texts=0;
                 while ((line = in.readLine()) != null) {
                     if (line.startsWith("#")) {
@@ -67,8 +67,7 @@
                         if (matcher.find()) {
                             fold = Math.abs(matcher.group(1).hashCode()) % folds + 1;
                         }
-                        int finalFold = fold;
-                        slidingWindowQueue = new SlidingWindowQueue(ngram_size, s -> FoldedEntry.incr(map, s, finalFold));
+                        slidingWindowQueue.reset(fold);
                         texts++;
                         continue;
                     }