Try to solve problems with early closed readers in long upsert transactions
Change-Id: I12704eeabef0c346d1b13a9cab384a2110b97ee9
diff --git a/Changes b/Changes
index 571803c..c851d6d 100644
--- a/Changes
+++ b/Changes
@@ -1,6 +1,8 @@
-0.59.1 2020-02-04
+0.59.1 2020-04-08
- [bugfix] Fix bug in classed group queries (diewald)
- [bugfix] Fix bug in segments with negated components (diewald)
+ - [bugfix] Try to fix problem with early closed readers
+ in upsert transactions (diewald)
0.59.0 2019-11-28
- [bugfix] Fix offset retrieval in concurrent searches
diff --git a/src/main/java/de/ids_mannheim/korap/KrillIndex.java b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
index 8efb926..49c4d7d 100644
--- a/src/main/java/de/ids_mannheim/korap/KrillIndex.java
+++ b/src/main/java/de/ids_mannheim/korap/KrillIndex.java
@@ -142,7 +142,9 @@
// TODO: Use configuration instead.
// Last line of defense against DOS
- private int autoCommit = 500;
+ // Keep in mind - upsert requires 2 commits
+ // and isn't atomic
+ private int autoCommit = 10000;
private String version = "Unknown";
private String name = "Unknown";
@@ -240,11 +242,12 @@
* @return The {@link IndexReader} object.
*/
public IndexReader reader () {
- // Todo: Maybe use DirectoryReader.openIfChanged(DirectoryReader)
+ // Todo: Maybe use DirectoryReader.openIfChanged(DirectoryReader)
if (!readerOpen)
this.openReader();
if (!readerOpen)
return null;
+
return this.reader;
};
@@ -300,7 +303,6 @@
// Open index reader
private void openReader () {
if (readerOpen) {
- // this.reader = DirectoryReader.openIfChanged(this.reader)
return;
};
@@ -376,6 +378,7 @@
* @throws IOException
*/
public void commit () throws IOException {
+ log.info("Internal committing index ... ");
this.writer().commit();
commitCounter = 0;
this.closeReader();
@@ -436,53 +439,72 @@
try {
// Iterate over all atomic indices and find the matching document
- if (this.reader() != null) {
+ UPSERT:
+ while (true) {
+
+ if (this.reader() != null) {
- for (LeafReaderContext atomic : this.reader().leaves()) {
-
- // Retrieve the single document of interest
- DocIdSet filterSet = filter.getDocIdSet(
- atomic,
- atomic.reader().getLiveDocs());
+ for (LeafReaderContext atomic : this.reader().leaves()) {
- DocIdSetIterator filterIterator = filterSet.iterator();
+ // The reader is closed
+ /*
+ if (this.reader.getRefCount() == 0) {
- if (filterIterator == null)
- continue;
-
- // Go to the matching doc - and remember its ID
- int localDocID = filterIterator.nextDoc();
-
- if (localDocID == DocIdSetIterator.NO_MORE_DOCS)
- continue;
-
- // We've found the correct document! Hurray!
- if (DEBUG)
- log.trace("We've found a matching document");
-
- // TODO: Probably use
- // document(int docID, StoredFieldVisitor visitor)
- Document storedDoc = atomic.reader().document(localDocID);
-
- // Document is loadable
- if (storedDoc != null) {
- IndexableField indexCreationField =
- storedDoc.getField("indexCreationDate");
-
- if (indexCreationField == null) {
- indexCreationDate = current;
- }
- else {
- indexCreationDate = new KrillDate(
- indexCreationField.numericValue().toString()
- );
+ // Retry update
+ // System.err.println("Retry update!");
+ break;
};
+ */
+
+ // Retrieve the single document of interest
+ DocIdSet filterSet = filter.getDocIdSet(
+ atomic,
+ atomic.reader().getLiveDocs());
+
+ DocIdSetIterator filterIterator = filterSet.iterator();
+
+ if (filterIterator == null) {
+ continue;
+ };
+
+ // Go to the matching doc - and remember its ID
+ int localDocID = filterIterator.nextDoc();
+
+ if (localDocID == DocIdSetIterator.NO_MORE_DOCS) {
+ continue;
+ };
+
+ // We've found the correct document! Hurray!
+ if (DEBUG)
+ log.trace("We've found a matching document");
+
+ // TODO: Probably use
+ // document(int docID, StoredFieldVisitor visitor)
+ Document storedDoc = atomic.reader().document(localDocID);
+
+ // Document is loadable
+ if (storedDoc != null) {
+ IndexableField indexCreationField =
+ storedDoc.getField("indexCreationDate");
+
+ if (indexCreationField == null) {
+ indexCreationDate = current;
+ }
+ else {
+ indexCreationDate = new KrillDate(
+ indexCreationField.numericValue().toString()
+ );
+ };
+ };
+
+ this.delDocs("textSigle", textSigle);
+ break UPSERT;
};
- this.delDocs("textSigle", textSigle);
+ }
+ else {
+ log.warn("Reader is null");
};
- }
- else {
- log.warn("Reader is null");
+ break;
};
}