Improved database collector
diff --git a/src/main/java/de/ids_mannheim/korap/KorapIndex.java b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
index 12a6a76..8d5b93d 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
@@ -1027,12 +1027,11 @@
* Search in the index.
*/
public KorapResult search (SpanQuery query) {
- return this.search(new KorapCollection(this), new KorapSearch(query));
+ return this.search(new KorapSearch(query));
};
public KorapResult search (SpanQuery query, short count) {
return this.search(
- new KorapCollection(this),
new KorapSearch(query).setCount(count)
);
};
@@ -1050,12 +1049,7 @@
KorapSearch ks = new KorapSearch(query);
ks.setStartIndex(startIndex).setCount(count);
ks.setContext(new SearchContext(leftTokenContext, leftContext, rightTokenContext, rightContext));
- return this.search(new KorapCollection(this), ks);
- };
-
- public KorapResult search (KorapSearch ks) {
- // TODO: This might leak
- return this.search(new KorapCollection(this), ks);
+ return this.search(ks);
};
@Deprecated
@@ -1068,17 +1062,28 @@
boolean rightTokenContext,
short rightContext) {
KorapSearch ks = new KorapSearch(query);
- ks.setContext(new SearchContext(leftTokenContext, leftContext, rightTokenContext, rightContext));
- return this.search(collection, ks);
+ ks.setContext(
+ new SearchContext(
+ leftTokenContext,
+ leftContext,
+ rightTokenContext,
+ rightContext
+ )
+ );
+ ks.setCollection(collection);
+ return this.search(ks);
};
// To be honest - the collection is already part of the KorapSearch!
- public KorapResult search (KorapCollection collection, KorapSearch ks) {
+ public KorapResult search (KorapSearch ks) {
if (DEBUG)
log.trace("Start search");
this.termContexts = new HashMap<Term, TermContext>();
+ KorapCollection collection = ks.getCollection();
+ collection.setIndex(this);
+
// Get the spanquery from the KorapSearch object
SpanQuery query = ks.getQuery();
@@ -1275,12 +1280,13 @@
// Collect matches
- public MatchCollector collect (KorapCollection collection,
- KorapSearch ks,
- MatchCollector mc) {
+ public MatchCollector collect (KorapSearch ks, MatchCollector mc) {
if (DEBUG)
log.trace("Start collecting");
+ KorapCollection collection = ks.getCollection();
+ collection.setIndex(this);
+
// Init term context
this.termContexts = new HashMap<Term, TermContext>();
@@ -1377,7 +1383,7 @@
log.warn(e.getLocalizedMessage());
};
- mc.commit();
+ mc.close();
return mc;
};
};
diff --git a/src/main/java/de/ids_mannheim/korap/KorapSearch.java b/src/main/java/de/ids_mannheim/korap/KorapSearch.java
index c882ae4..7b2ff73 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapSearch.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapSearch.java
@@ -289,7 +289,7 @@
};
this.getCollection().setIndex(ki);
- KorapResult kr = ki.search(this.getCollection(), this);
+ KorapResult kr = ki.search(this);
kr.setRequest(this.request);
if (this.warning != null)
kr.addWarning(this.warning);
diff --git a/src/main/java/de/ids_mannheim/korap/index/MatchCollector.java b/src/main/java/de/ids_mannheim/korap/index/MatchCollector.java
index e1e93e3..ca3f056 100644
--- a/src/main/java/de/ids_mannheim/korap/index/MatchCollector.java
+++ b/src/main/java/de/ids_mannheim/korap/index/MatchCollector.java
@@ -26,7 +26,7 @@
};
public void commit() {};
-
+ public void close() {};
/*
* The following methods are shared and should be used from KorapResult
diff --git a/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java b/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
index b7d212e..0dbe73c 100644
--- a/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
+++ b/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
@@ -1,6 +1,11 @@
package de.ids_mannheim.korap.index.collector;
import de.ids_mannheim.korap.KorapMatch;
import de.ids_mannheim.korap.index.MatchCollector;
+import com.fasterxml.jackson.annotation.*;
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import javax.sql.DataSource;
+import java.sql.SQLException;
import java.util.*;
public class MatchCollectorDB extends MatchCollector {
@@ -9,35 +14,79 @@
Todo: In case there are multiple threads searching,
the list should be synchrinized Collections.synchronizedList()
*/
+ private String databaseType;
private List matchCollector;
private int bufferSize;
- private int doccollect;
+ private int docCollect;
+ private String resultID;
- private String tableName;
+ private Connection connection;
+ private PreparedStatement prepared;
/*
* Create a new collector for database connections
*/
- public MatchCollectorDB (int bufferSize, String tableName) {
+ public MatchCollectorDB (int bufferSize, String resultID) {
this.bufferSize = bufferSize;
- this.tableName = tableName;
+ this.resultID = resultID;
this.matchCollector = new ArrayList<int[]>(bufferSize + 2);
};
/*
* Add matches till the bufferSize exceeds - then commit to the database.
*/
- public void add (int uniqueDocID, int matchcount) {
+ public void add (int UID, int matchCount) {
this.incrTotalResultDocs(1);
- this.incrTotalResults(matchcount);
- this.matchCollector.add(new int[]{uniqueDocID, matchcount});
- if (this.doccollect++ > bufferSize)
+ this.incrTotalResults(matchCount);
+ this.matchCollector.add(new int[]{UID, matchCount});
+
+ if (this.docCollect++ > bufferSize)
this.commit();
};
- public void commit () {
-
+ @JsonIgnore
+ public void setDatabaseType (String type) {
+ this.databaseType = type;
+ };
+
+ @JsonIgnore
+ public String getDatabaseType () {
+ return this.databaseType;
+ };
+
+ @JsonIgnore
+ public void openConnection (String type, DataSource ds) throws SQLException {
+ this.setDatabaseType(type);
+ this.connection = ds.getConnection();
+ this.connection.setAutoCommit(false);
+
+ // Create prepared statement for multiple requests
+
+ /*
+ this.prepared = this.conn.prepareStatement(
+ "INSERT INTO people VALUES (?, ?);"
+ );
+
+ Only prepare if commit > buffersize!
+Difference between mariadb and sqlite!
+ */
+
+ };
+
+ public void commit () {
+
+ /*
+ */
this.matchCollector.clear();
- this.doccollect = 0;
+ this.docCollect = 0;
+ };
+
+ public void close () {
+ this.commit();
+ try {
+ this.connection.close();
+ }
+ catch (SQLException e) {
+ };
};
};
diff --git a/src/main/java/de/ids_mannheim/korap/server/Resource.java b/src/main/java/de/ids_mannheim/korap/server/Resource.java
index eb0119a..818b6c3 100644
--- a/src/main/java/de/ids_mannheim/korap/server/Resource.java
+++ b/src/main/java/de/ids_mannheim/korap/server/Resource.java
@@ -208,12 +208,11 @@
*
* @param text_id
*/
- /*
@POST
@Path("/collect/{resultID}")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
- public String find (String json, @Context UriInfo uri) {
+ public String collect (String json, @Context UriInfo uri) {
// Get index
KorapIndex index = KorapNode.getIndex();
@@ -225,14 +224,17 @@
index.getVersion()
).setError(601, "Unable to find index").toJSON();
+
+ return "";
+
// Get the database
// Create a database based matchcollector
// TODO: Only search in self documents (REPLICATION FTW!)
- MatchCollector result = index.collect(KorapCollection, KorapSearch, MatchCollector);
-
+ //MatchCollector result = index.collect(KorapCollection, KorapSearch, MatchCollector);
+ /*
// Build Collection based on a list of uids
List<String> uids = qp.get("uid");
@@ -259,8 +261,8 @@
KorapNode.getName(),
index.getVersion()
).setError(601, "Unable to find index").toJSON();
- };
*/
+ };
diff --git a/src/test/java/de/ids_mannheim/korap/server/TestDatabase.java b/src/test/java/de/ids_mannheim/korap/server/TestDatabase.java
index 04e7603..80276cf 100644
--- a/src/test/java/de/ids_mannheim/korap/server/TestDatabase.java
+++ b/src/test/java/de/ids_mannheim/korap/server/TestDatabase.java
@@ -10,9 +10,7 @@
import java.sql.Statement;
import java.sql.SQLException;
-/*
- bitbucket.org/xerial/sqlite-jdbc
-*/
+import com.mchange.v2.c3p0.*;
import org.junit.After;
import org.junit.Before;
@@ -66,6 +64,64 @@
};
@Test
+ public void TestDatabasePool () throws Exception {
+ ComboPooledDataSource cpds = new ComboPooledDataSource();
+ // Connect to a temporary file instead of a in-memory file
+ cpds.setDriverClass("org.sqlite.JDBC");
+ cpds.setJdbcUrl("jdbc:sqlite:");
+ // cpds.setUser(xxx);
+ // cpds.setPassword(xxx);
+ cpds.setMaxStatements(100);
+ // cpds.autoCommitOnClose(true);
+ // cpds.aquireRetryDelay(100);
+
+ // This is part of the match collector
+ this.conn = cpds.getConnection();
+ this.stat = conn.createStatement();
+ stat.executeUpdate(
+ "CREATE TABLE IF NOT EXISTS result_a (text_id INTEGER, match_count INTEGER);"
+ );
+ conn.setAutoCommit(false);
+ PreparedStatement prep = this.conn.prepareStatement(
+ "INSERT INTO result_a VALUES (?, ?);"
+ );
+ prep.setInt(1, 5);
+ prep.setInt(2, 8000);
+ prep.addBatch();
+ prep.executeBatch();
+
+ ResultSet rs = stat.executeQuery("SELECT * FROM result_a;");
+
+ rs.next();
+
+ assertEquals(rs.getInt("text_id"), 5);
+ assertEquals(rs.getInt("match_count"), 8000);
+
+ rs.close();
+
+ this.conn.close();
+
+
+ MatchCollectorDB mc = new MatchCollectorDB(2000, "result_a");
+ mc.openConnection("sqlite", cpds);
+
+
+ mc.add(9, 5000);
+ mc.add(12, 6785);
+ mc.add(39, 56576);
+
+ mc.close();
+
+ cpds.close();
+
+ /*
+ this.stat = this.conn.createStatement();
+ stat.executeUpdate("CREATE TABLE IF NOT EXISTS result_a (text_id INTEGER, match_count INTEGER);");
+ */
+ };
+
+
+ @Test
public void TestMatchCollectorDB () throws Exception {
MatchCollector mc = new MatchCollectorDB(2000, "matchXYZ");
mc.add(5,7);