Fixed database tests
diff --git a/CHANGES b/CHANGES
index febbab7..572f9f6 100644
--- a/CHANGES
+++ b/CHANGES
@@ -1,4 +1,4 @@
-0.44 2014-09-27
+0.44 2014-09-28
- [feature] Match collector using database for distributed search (diewald)
- [bugfix] Unified boundary handling and prevent nullpointer
exceptions on parsing (diewald)
@@ -7,6 +7,7 @@
- [performance] Updated Lucene dependency from 4.3.1 to 4.5.1 (diewald)
- [performance] Updated Jackson dependency from 2.2.2 to 2.4.0 (diewald)
- [bugfix] Return matches correctly with JSON (diewald)
+ - [bugfix] Repare database tests working with temporary databases (diewald)
0.43 2014-09-23
- [cleanup] Made a lot of stuff rely on KorapResponse (diewald)
diff --git a/src/main/java/de/ids_mannheim/korap/KorapIndex.java b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
index c98b06d..340e527 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
@@ -484,12 +484,9 @@
};
long docCount = 0;
- // System.err.println("CHECK");
int i = 1;
for (AtomicReaderContext atomic : this.reader().leaves()) {
- // System.err.println("READER" + i + "a-" + docCount);
docCount += collection.bits(atomic).cardinality();
- // System.err.println("READER" + i + "b-" + docCount);
i++;
};
return docCount;
@@ -497,7 +494,6 @@
// Create search term
Term term = new Term(field, "-:" + type);
- // System.err.println(">> Search for -:" + type + " in " + field);
long occurrences = 0;
try {
diff --git a/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java b/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
index 62bc4c7..10650dd 100644
--- a/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
+++ b/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
@@ -18,15 +18,15 @@
private final static Logger log = LoggerFactory.getLogger(KorapNode.class);
/*
- Todo: In case there are multiple threads searching,
- the list should be synchrinized Collections.synchronizedList()
+ * Todo: In case there are multiple threads searching,
+ * the list should be synchrinized Collections.synchronizedList()
*/
private String databaseType;
private List matchCollector;
private int bufferSize, docCollect;
private String resultID;
- // private Connection connection;
+ // private Connection connection;
private DataSource pool;
private Connection connection;
private PreparedStatement prepared;
@@ -64,22 +64,27 @@
};
@JsonIgnore
+ public void setDBPool (String type, DataSource ds, Connection conn) throws SQLException {
+ this.setDatabaseType(type);
+ this.connection = conn;
+ this.pool = ds;
+ };
+
+
+ @JsonIgnore
public void setDBPool (String type, DataSource ds) throws SQLException {
this.setDatabaseType(type);
this.pool = ds;
-
- // Create prepared statement for multiple requests
-
- /*
- this.prepared = this.conn.prepareStatement(
- "INSERT INTO people VALUES (?, ?);"
- );
-
- Only prepare if commit > buffersize!
-Difference between mariadb and sqlite!
- */
-
};
+ /*
+ Create prepared statement for multiple requests
+ this.prepared = this.conn.prepareStatement(
+ "INSERT INTO people VALUES (?, ?);"
+ );
+ Only prepare if commit > buffersize!
+ Difference between mariadb and sqlite!
+ */
+
/* TODO: Ensure the commit was successful! */
public void commit () {
@@ -87,23 +92,20 @@
return;
try {
- // This should be heavily optimized! It's aweful!
/*
+ * This should be heavily optimized! It's aweful!
* ARGHHHHHHH!
*/
- if (this.connection == null)
+ if (this.connection.isClosed())
this.connection = this.pool.getConnection();
- // TODO: Create a BEGIN ... COMMIT Transaction
- // connection.setAutoCommit(true);
-
StringBuilder sb = new StringBuilder();
- sb.append("INSERT INTO ");
- sb.append(this.resultID);
- sb.append(" (text_id, match_count) ");
+ sb.append("INSERT INTO ")
+ .append(this.resultID)
+ .append(" (text_id, match_count) ");
- // SQLite insertion idiom
+ // SQLite batch insertion idiom
if (this.getDatabaseType().equals("sqlite")) {
for (int i = 1; i < this.docCollect; i++) {
sb.append("SELECT ?, ? UNION ");
@@ -114,7 +116,7 @@
sb.append("SELECT ?, ?");
}
- // MySQL insertion idiom
+ // MySQL batch insertion idiom
else if (this.getDatabaseType().equals("mysql")) {
sb.append(" VALUES ");
for (int i = 1; i < this.docCollect; i++) {
@@ -122,52 +124,59 @@
};
sb.append("(?,?)");
}
+
+ // Unknown idiom
else {
log.error("Unsupported Database type");
return;
};
- // System.err.println(sb.toString());
-
- PreparedStatement prep = connection.prepareStatement(sb.toString());
+ // Prepare statement based on the string
+ PreparedStatement prep = this.connection.prepareStatement(sb.toString());
int i = 1;
ListIterator li = this.matchCollector.listIterator();
while (li.hasNext()) {
int[] v = (int[]) li.next();
- // System.err.println("Has " + i + ":" + v[0]);
prep.setInt(i++, v[0]);
- // System.err.println("Has " + i + ":" + v[1]);
prep.setInt(i++, v[1]);
- // System.err.println("-");
};
- // System.err.println(sb.toString());
-
prep.addBatch();
prep.executeBatch();
- // connection.setAutoCommit(false);
- // connection.close();
- this.matchCollector.clear();
- this.docCollect = 0;
+ this.connection.commit();
}
+
+ // An SQL error occured ...
catch (SQLException e) {
- this.matchCollector.clear();
- this.docCollect = 0;
- System.err.println("Error: " + e.getLocalizedMessage());
log.error(e.getLocalizedMessage());
};
+
+ this.matchCollector.clear();
+ this.docCollect = 0;
return;
};
+ /*
+ * Close collector and connection
+ */
public void close () {
this.commit();
- /*
try {
this.connection.close();
}
- catch (SQLException e) {
- };
- */
+ catch (SQLException e) {
+ log.warn(e.getLocalizedMessage());
+ }
+ };
+
+ /*
+ * Close collector and probably connection
+ */
+ public void close (boolean close) {
+ if (close)
+ this.close();
+ else
+ this.commit();
};
};
diff --git a/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java b/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
index 6e2d967..dbcbdd7 100644
--- a/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
+++ b/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
@@ -23,6 +23,7 @@
private String errstr, msg, version, node, listener;
private int err, unstaged;
private int totalResults;
+ private long totalTexts;
private String benchmark;
public KorapResponse (String node, String version) {
@@ -100,6 +101,17 @@
return this;
};
+ public KorapResponse setTotalTexts (long i) {
+ this.totalTexts = i;
+ return this;
+ };
+
+ public long getTotalTexts() {
+ return this.totalTexts;
+ };
+
+
+
public KorapResponse setTotalResults (int i) {
this.totalResults = i;
return this;
diff --git a/src/main/java/de/ids_mannheim/korap/server/Resource.java b/src/main/java/de/ids_mannheim/korap/server/Resource.java
index ba8d3e9..97d6973 100644
--- a/src/main/java/de/ids_mannheim/korap/server/Resource.java
+++ b/src/main/java/de/ids_mannheim/korap/server/Resource.java
@@ -42,6 +42,7 @@
import org.slf4j.LoggerFactory;
import java.sql.SQLException;
+import java.sql.Connection;
/*
http://www.vogella.com/tutorials/REST/article.html
@@ -76,13 +77,22 @@
};
@GET
- @Path("/info")
@Produces(MediaType.APPLICATION_JSON)
public String info () {
KorapIndex index = KorapNode.getIndex();
KorapResponse kresp = new KorapResponse(KorapNode.getName(), index.getVersion());
kresp.setListener(KorapNode.getListener());
- return kresp.setMsg("Up and running!").toJSON();
+ long texts = -1;
+ try {
+ texts = index.numberOf("documents");
+ }
+ catch (IOException e) {
+ log.error(e.getLocalizedMessage());
+ };
+
+ return kresp.setTotalTexts(texts)
+ .setMsg("Up and running!")
+ .toJSON();
};
@@ -247,7 +257,8 @@
// Get the database
try {
MatchCollectorDB mc = new MatchCollectorDB(1000, "Res_" + resultID);
- mc.setDBPool("mysql", KorapNode.getDBPool());
+ Connection conn = KorapNode.getDBPool().getConnection();
+ mc.setDBPool("mysql", KorapNode.getDBPool(), conn);
// TODO: Only search in self documents (REPLICATION FTW!)
diff --git a/src/main/resources/index.properties b/src/main/resources/index.properties
index fa1fd47..f1d4d58 100644
--- a/src/main/resources/index.properties
+++ b/src/main/resources/index.properties
@@ -1 +1,3 @@
lucene.index.version = ${project.version}
+lucene.index.commit.count = 134217000
+lucene.index.commit.log = log/korap.commit.log
diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties
index 7c2b69a..48c48d1 100644
--- a/src/main/resources/log4j.properties
+++ b/src/main/resources/log4j.properties
@@ -1,6 +1,6 @@
## logger file can be used with
-#log4j.rootLogger = DEBUG, stdout
+log4j.rootLogger = ERROR, stdout
# Spans:
#log4j.logger.de.ids_mannheim.korap.query.spans.ElementSpans = TRACE, stdout
diff --git a/src/main/resources/server.properties.info b/src/main/resources/server.properties.info
new file mode 100644
index 0000000..2a392e8
--- /dev/null
+++ b/src/main/resources/server.properties.info
@@ -0,0 +1,11 @@
+# Lucene Backend properties
+lucene.properties = true
+lucene.indexDir = [PATH TO INDEX DIRECTORY]
+lucene.node.name = [UNIQUE NODE NAME]
+lucene.node.baseURI = [LISTEN-URL INCLUDING PORT]
+
+# Lucene Database properties
+lucene.db.class = org.mariadb.jdbc.Driver
+lucene.db.URL = jdbc:mysql://[DB_IP]:[DB_PORT]/[DB_NAME]
+lucene.db.pwd = [DB_PWD]
+lucene.db.user = [DB_USER]
diff --git a/src/test/java/de/ids_mannheim/korap/server/TestDatabase.java b/src/test/java/de/ids_mannheim/korap/server/TestDatabase.java
index ec06631..009e50e 100644
--- a/src/test/java/de/ids_mannheim/korap/server/TestDatabase.java
+++ b/src/test/java/de/ids_mannheim/korap/server/TestDatabase.java
@@ -16,7 +16,7 @@
import org.junit.Before;
import org.junit.Test;
import org.junit.Ignore;
-import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.*;
public class TestDatabase {
@@ -69,16 +69,17 @@
* temporary dbs - should be improved
*/
- @Ignore
+ @Test
public void TestDatabasePool () throws Exception {
ComboPooledDataSource cpds = new ComboPooledDataSource();
// Connect to a temporary file instead of a in-memory file
cpds.setDriverClass("org.sqlite.JDBC");
- cpds.setJdbcUrl("jdbc:sqlite:hui");
+ cpds.setJdbcUrl("jdbc:sqlite:");
cpds.setMaxStatements(100);
// This is part of the match collector
this.conn = cpds.getConnection();
+ conn.setAutoCommit(false);
this.stat = conn.createStatement();
stat.executeUpdate(
"CREATE TABLE IF NOT EXISTS result_a (text_id INTEGER, match_count INTEGER);"
@@ -93,49 +94,57 @@
prep.executeBatch();
ResultSet rs = stat.executeQuery("SELECT * FROM result_a;");
-
rs.next();
-
assertEquals(rs.getInt("text_id"), 5);
assertEquals(rs.getInt("match_count"), 8000);
-
rs.close();
- // this.conn.close();
-
MatchCollectorDB mc = new MatchCollectorDB(2000, "result_a");
- mc.setDBPool("sqlite", cpds);
+ mc.setDBPool("sqlite", cpds, this.conn);
mc.add(9, 5000);
mc.add(12, 6785);
mc.add(39, 56576);
- mc.close();
+ mc.close(false);
- /*
- this.stat = this.conn.createStatement();
- stat.executeUpdate("CREATE TABLE IF NOT EXISTS result_a (text_id INTEGER, match_count INTEGER);");
- */
+ rs = stat.executeQuery("SELECT * FROM result_a;");
+ assertTrue(rs.next());
+ assertEquals(rs.getInt("text_id"), 5);
+ assertEquals(rs.getInt("match_count"), 8000);
+ rs.next();
+ assertEquals(rs.getInt("text_id"), 9);
+ assertEquals(rs.getInt("match_count"), 5000);
+ rs.next();
+ assertEquals(rs.getInt("text_id"), 12);
+ assertEquals(rs.getInt("match_count"), 6785);
+ rs.next();
+ assertEquals(rs.getInt("text_id"), 39);
+ assertEquals(rs.getInt("match_count"), 56576);
+
+ rs.close();
};
- @Ignore
- public void TestDatabasePoolConnector () throws Exception {
+ @Test
+ public void TestDatabasePoolCollector () throws Exception {
ComboPooledDataSource cpds = new ComboPooledDataSource();
// Connect to a temporary file instead of a in-memory file
cpds.setDriverClass("org.sqlite.JDBC");
- cpds.setJdbcUrl("jdbc:sqlite:hui");
+ cpds.setJdbcUrl("jdbc:sqlite:");
cpds.setMaxStatements(100);
// This is part of the match collector
conn = cpds.getConnection();
- stat = conn.createStatement();
- // conn.setAutoCommit(true);
+ conn.setAutoCommit(false);
+ Statement stat = conn.createStatement();
stat.executeUpdate(
- "CREATE TABLE matchXYZ (text_id INTEGER, match_count INTEGER);"
+ "CREATE TABLE IF NOT EXISTS matchXYZ (text_id INTEGER, match_count INTEGER);"
);
+ conn.commit();
+ stat.close();
MatchCollectorDB mc = new MatchCollectorDB(3, "matchXYZ");
- mc.setDBPool("sqlite", cpds);
+ mc.setDBPool("sqlite", cpds, conn);
mc.add(9, 5000);
mc.add(12, 6785);
@@ -148,14 +157,41 @@
// Second commit
mc.add(94, 456);
- mc.close();
+ mc.close(false);
// Final commit
// conn = cpds.getConnection();
stat = conn.createStatement();
ResultSet rs = stat.executeQuery("SELECT count('*') AS num FROM matchXYZ;");
- rs.next();
+
assertEquals(7, rs.getInt("num"));
+
+ rs = stat.executeQuery("SELECT text_id, match_count FROM matchXYZ;");
+ assertTrue(rs.next());
+
+ assertEquals(rs.getInt("text_id"), 9);
+ assertEquals(rs.getInt("match_count"), 5000);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt("text_id"), 12);
+ assertEquals(rs.getInt("match_count"), 6785);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt("text_id"), 39);
+ assertEquals(rs.getInt("match_count"), 56576);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt("text_id"), 45);
+ assertEquals(rs.getInt("match_count"), 5000);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt("text_id"), 67);
+ assertEquals(rs.getInt("match_count"), 6785);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt("text_id"), 81);
+ assertEquals(rs.getInt("match_count"), 56576);
+ assertTrue(rs.next());
+ assertEquals(rs.getInt("text_id"), 94);
+ assertEquals(rs.getInt("match_count"), 456);
+ assertFalse(rs.next());
+
+ stat.close();
};
@Test