Fixed database tests
diff --git a/src/main/java/de/ids_mannheim/korap/KorapIndex.java b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
index c98b06d..340e527 100644
--- a/src/main/java/de/ids_mannheim/korap/KorapIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KorapIndex.java
@@ -484,12 +484,9 @@
};
long docCount = 0;
- // System.err.println("CHECK");
int i = 1;
for (AtomicReaderContext atomic : this.reader().leaves()) {
- // System.err.println("READER" + i + "a-" + docCount);
docCount += collection.bits(atomic).cardinality();
- // System.err.println("READER" + i + "b-" + docCount);
i++;
};
return docCount;
@@ -497,7 +494,6 @@
// Create search term
Term term = new Term(field, "-:" + type);
- // System.err.println(">> Search for -:" + type + " in " + field);
long occurrences = 0;
try {
diff --git a/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java b/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
index 62bc4c7..10650dd 100644
--- a/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
+++ b/src/main/java/de/ids_mannheim/korap/index/collector/MatchCollectorDB.java
@@ -18,15 +18,15 @@
private final static Logger log = LoggerFactory.getLogger(KorapNode.class);
/*
- Todo: In case there are multiple threads searching,
- the list should be synchrinized Collections.synchronizedList()
+ * Todo: In case there are multiple threads searching,
+ * the list should be synchrinized Collections.synchronizedList()
*/
private String databaseType;
private List matchCollector;
private int bufferSize, docCollect;
private String resultID;
- // private Connection connection;
+ // private Connection connection;
private DataSource pool;
private Connection connection;
private PreparedStatement prepared;
@@ -64,22 +64,27 @@
};
@JsonIgnore
+ public void setDBPool (String type, DataSource ds, Connection conn) throws SQLException {
+ this.setDatabaseType(type);
+ this.connection = conn;
+ this.pool = ds;
+ };
+
+
+ @JsonIgnore
public void setDBPool (String type, DataSource ds) throws SQLException {
this.setDatabaseType(type);
this.pool = ds;
-
- // Create prepared statement for multiple requests
-
- /*
- this.prepared = this.conn.prepareStatement(
- "INSERT INTO people VALUES (?, ?);"
- );
-
- Only prepare if commit > buffersize!
-Difference between mariadb and sqlite!
- */
-
};
+ /*
+ Create prepared statement for multiple requests
+ this.prepared = this.conn.prepareStatement(
+ "INSERT INTO people VALUES (?, ?);"
+ );
+ Only prepare if commit > buffersize!
+ Difference between mariadb and sqlite!
+ */
+
/* TODO: Ensure the commit was successful! */
public void commit () {
@@ -87,23 +92,20 @@
return;
try {
- // This should be heavily optimized! It's aweful!
/*
+ * This should be heavily optimized! It's aweful!
* ARGHHHHHHH!
*/
- if (this.connection == null)
+ if (this.connection.isClosed())
this.connection = this.pool.getConnection();
- // TODO: Create a BEGIN ... COMMIT Transaction
- // connection.setAutoCommit(true);
-
StringBuilder sb = new StringBuilder();
- sb.append("INSERT INTO ");
- sb.append(this.resultID);
- sb.append(" (text_id, match_count) ");
+ sb.append("INSERT INTO ")
+ .append(this.resultID)
+ .append(" (text_id, match_count) ");
- // SQLite insertion idiom
+ // SQLite batch insertion idiom
if (this.getDatabaseType().equals("sqlite")) {
for (int i = 1; i < this.docCollect; i++) {
sb.append("SELECT ?, ? UNION ");
@@ -114,7 +116,7 @@
sb.append("SELECT ?, ?");
}
- // MySQL insertion idiom
+ // MySQL batch insertion idiom
else if (this.getDatabaseType().equals("mysql")) {
sb.append(" VALUES ");
for (int i = 1; i < this.docCollect; i++) {
@@ -122,52 +124,59 @@
};
sb.append("(?,?)");
}
+
+ // Unknown idiom
else {
log.error("Unsupported Database type");
return;
};
- // System.err.println(sb.toString());
-
- PreparedStatement prep = connection.prepareStatement(sb.toString());
+ // Prepare statement based on the string
+ PreparedStatement prep = this.connection.prepareStatement(sb.toString());
int i = 1;
ListIterator li = this.matchCollector.listIterator();
while (li.hasNext()) {
int[] v = (int[]) li.next();
- // System.err.println("Has " + i + ":" + v[0]);
prep.setInt(i++, v[0]);
- // System.err.println("Has " + i + ":" + v[1]);
prep.setInt(i++, v[1]);
- // System.err.println("-");
};
- // System.err.println(sb.toString());
-
prep.addBatch();
prep.executeBatch();
- // connection.setAutoCommit(false);
- // connection.close();
- this.matchCollector.clear();
- this.docCollect = 0;
+ this.connection.commit();
}
+
+ // An SQL error occured ...
catch (SQLException e) {
- this.matchCollector.clear();
- this.docCollect = 0;
- System.err.println("Error: " + e.getLocalizedMessage());
log.error(e.getLocalizedMessage());
};
+
+ this.matchCollector.clear();
+ this.docCollect = 0;
return;
};
+ /*
+ * Close collector and connection
+ */
public void close () {
this.commit();
- /*
try {
this.connection.close();
}
- catch (SQLException e) {
- };
- */
+ catch (SQLException e) {
+ log.warn(e.getLocalizedMessage());
+ }
+ };
+
+ /*
+ * Close collector and probably connection
+ */
+ public void close (boolean close) {
+ if (close)
+ this.close();
+ else
+ this.commit();
};
};
diff --git a/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java b/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
index 6e2d967..dbcbdd7 100644
--- a/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
+++ b/src/main/java/de/ids_mannheim/korap/server/KorapResponse.java
@@ -23,6 +23,7 @@
private String errstr, msg, version, node, listener;
private int err, unstaged;
private int totalResults;
+ private long totalTexts;
private String benchmark;
public KorapResponse (String node, String version) {
@@ -100,6 +101,17 @@
return this;
};
+ public KorapResponse setTotalTexts (long i) {
+ this.totalTexts = i;
+ return this;
+ };
+
+ public long getTotalTexts() {
+ return this.totalTexts;
+ };
+
+
+
public KorapResponse setTotalResults (int i) {
this.totalResults = i;
return this;
diff --git a/src/main/java/de/ids_mannheim/korap/server/Resource.java b/src/main/java/de/ids_mannheim/korap/server/Resource.java
index ba8d3e9..97d6973 100644
--- a/src/main/java/de/ids_mannheim/korap/server/Resource.java
+++ b/src/main/java/de/ids_mannheim/korap/server/Resource.java
@@ -42,6 +42,7 @@
import org.slf4j.LoggerFactory;
import java.sql.SQLException;
+import java.sql.Connection;
/*
http://www.vogella.com/tutorials/REST/article.html
@@ -76,13 +77,22 @@
};
@GET
- @Path("/info")
@Produces(MediaType.APPLICATION_JSON)
public String info () {
KorapIndex index = KorapNode.getIndex();
KorapResponse kresp = new KorapResponse(KorapNode.getName(), index.getVersion());
kresp.setListener(KorapNode.getListener());
- return kresp.setMsg("Up and running!").toJSON();
+ long texts = -1;
+ try {
+ texts = index.numberOf("documents");
+ }
+ catch (IOException e) {
+ log.error(e.getLocalizedMessage());
+ };
+
+ return kresp.setTotalTexts(texts)
+ .setMsg("Up and running!")
+ .toJSON();
};
@@ -247,7 +257,8 @@
// Get the database
try {
MatchCollectorDB mc = new MatchCollectorDB(1000, "Res_" + resultID);
- mc.setDBPool("mysql", KorapNode.getDBPool());
+ Connection conn = KorapNode.getDBPool().getConnection();
+ mc.setDBPool("mysql", KorapNode.getDBPool(), conn);
// TODO: Only search in self documents (REPLICATION FTW!)
diff --git a/src/main/resources/index.properties b/src/main/resources/index.properties
index fa1fd47..f1d4d58 100644
--- a/src/main/resources/index.properties
+++ b/src/main/resources/index.properties
@@ -1 +1,3 @@
lucene.index.version = ${project.version}
+lucene.index.commit.count = 134217000
+lucene.index.commit.log = log/korap.commit.log
diff --git a/src/main/resources/log4j.properties b/src/main/resources/log4j.properties
index 7c2b69a..48c48d1 100644
--- a/src/main/resources/log4j.properties
+++ b/src/main/resources/log4j.properties
@@ -1,6 +1,6 @@
## logger file can be used with
-#log4j.rootLogger = DEBUG, stdout
+log4j.rootLogger = ERROR, stdout
# Spans:
#log4j.logger.de.ids_mannheim.korap.query.spans.ElementSpans = TRACE, stdout
diff --git a/src/main/resources/server.properties.info b/src/main/resources/server.properties.info
new file mode 100644
index 0000000..2a392e8
--- /dev/null
+++ b/src/main/resources/server.properties.info
@@ -0,0 +1,11 @@
+# Lucene Backend properties
+lucene.properties = true
+lucene.indexDir = [PATH TO INDEX DIRECTORY]
+lucene.node.name = [UNIQUE NODE NAME]
+lucene.node.baseURI = [LISTEN-URL INCLUDING PORT]
+
+# Lucene Database properties
+lucene.db.class = org.mariadb.jdbc.Driver
+lucene.db.URL = jdbc:mysql://[DB_IP]:[DB_PORT]/[DB_NAME]
+lucene.db.pwd = [DB_PWD]
+lucene.db.user = [DB_USER]