Fix sentence splitting around german opening/closing quotes
Change-Id: Ie2b5f2cc293127273947f3225fe9a50b54986ea6
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 5d78cba..6fbcbe2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
# Changelog
+## 2.5.0 [2026-09-03]
+* Fixed sentence splitting behaviour for opening/closing german double quotes
+* Added horizontal ellipsis as a sentence boundary
+* Join closing quotes in sentence boundaries
## 2.4.2 [2026-06-16]
diff --git a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/DerekoDfaTokenizer.jflex b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/DerekoDfaTokenizer.jflex
index afad9ee..2c248ff 100644
--- a/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/DerekoDfaTokenizer.jflex
+++ b/src/main/jpc/jflex/de/ids_mannheim/korap/tokenizer/DerekoDfaTokenizer.jflex
@@ -224,6 +224,17 @@
tokenStringBuffer.append(" ");
}
if (isSentenceBound(s.getType()) || (i == spanList.size() - 1)) {
+ while (i + 1 < spanList.size() && isClosingQuote(spanList.get(i + 1).getType())) {
+ i++;
+ s = spanList.get(i);
+ if (printOffsets) {
+ tokenStringBuffer.append(s.getStart())
+ .append(" ")
+ .append(s.getEnd());
+ if (i < spanList.size() - 1)
+ tokenStringBuffer.append(" ");
+ }
+ }
sentenceStringBuffer.append(sentenceStart)
.append(" ")
.append(s.getEnd());
@@ -297,7 +308,10 @@
if (tokens.length > 0)
tokens[0].getStart();
for (int i = 0; i < tokens.length; i++) {
- if (tokens[i].getType().matches("^[.?!]+$") || i == tokens.length - 1) {
+ if (isSentenceBound(tokens[i].getType()) || i == tokens.length - 1) {
+ while (i + 1 < tokens.length && isClosingQuote(tokens[i + 1].getType())) {
+ i++;
+ }
sentences.add(new Span(sentenceStart, tokens[i].getEnd(), s.substring(sentenceStart, tokens[i].getEnd())));
if (i < tokens.length - 1) {
sentenceStart = tokens[i + 1].getStart();
@@ -326,7 +340,11 @@
}
public boolean isSentenceBound(String s) {
- return s.matches("^[.?!]+$");
+ return s.matches("^[.?!\u2026]+$");
+ }
+
+ private boolean isClosingQuote(String tokenText) {
+ return tokenText.equals("\u201C") || tokenText.equals("\u201D") || tokenText.equals("''");
}
final Span currentToken(String normalizedValue) {
@@ -886,6 +904,8 @@
‘ { yybegin(OPEN_QUOTE); return currentToken("`"); }
’ { yybegin(YYINITIAL); return currentToken("'"); }
<OPEN_QUOTE>\" { yybegin(YYINITIAL); return currentToken("''"); }
+<OPEN_QUOTE>“ { yybegin(YYINITIAL); return currentToken("''"); }
+„ { yybegin(OPEN_QUOTE); return currentToken("``"); }
“ { yybegin(YYINITIAL); return currentToken("``"); }
” { yybegin(YYINITIAL); return currentToken("''"); }
\"/.*{ALPHANUM}+ { yybegin(OPEN_QUOTE); return currentToken("``"); }
diff --git a/src/test/java/de/ids_mannheim/korap/tokenizer/SentenceSplitterTest.java b/src/test/java/de/ids_mannheim/korap/tokenizer/SentenceSplitterTest.java
index f22d078..58051ae 100644
--- a/src/test/java/de/ids_mannheim/korap/tokenizer/SentenceSplitterTest.java
+++ b/src/test/java/de/ids_mannheim/korap/tokenizer/SentenceSplitterTest.java
@@ -5,6 +5,7 @@
import org.junit.Assume;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
+import opennlp.tools.util.Span;
import static org.junit.Assert.assertEquals;
@@ -109,4 +110,125 @@
assertEquals("\"Hast Du nicht gehört???\"", sentences[1]);
assertEquals(sentences.length, 2);
}
+
+ @Test
+ public void testSentSplitterGermanNestedQuotes () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String input = "Worum es gehe, erkundigte sich Nicole bei einem der Krankenpfleger. „Was ist mit dem Baby?“ „Blutdruck normal, 110 zu 75. Puls 62, leicht sinkend …“ Während der Sanitäter die Daten im Telegrammstil wiedergab, sah Nicole auf die bewusstlose Patientin hinunter. Ihr Gesicht,";
+
+ String[] sentences = tok.sentDetect(input);
+
+ assertEquals(5, sentences.length);
+ assertEquals("Worum es gehe, erkundigte sich Nicole bei einem der Krankenpfleger.", sentences[0]);
+ assertEquals("„Was ist mit dem Baby?“", sentences[1]);
+ assertEquals("„Blutdruck normal, 110 zu 75. Puls 62, leicht sinkend …“", sentences[2]);
+ assertEquals("Während der Sanitäter die Daten im Telegrammstil wiedergab, sah Nicole auf die bewusstlose Patientin hinunter.", sentences[3]);
+ assertEquals("Ihr Gesicht,", sentences[4]);
+ }
+
+ @Test
+ public void testSentSplitterGermanQuoteWithPeriod () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("„Das ist gut.“ Er ging.");
+ assertEquals(2, sentences.length);
+ assertEquals("„Das ist gut.“", sentences[0]);
+ assertEquals("Er ging.", sentences[1]);
+ }
+
+ @Test
+ public void testSentSplitterGermanQuoteWithExclamation () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("„Halt!“ Er blieb stehen.");
+ assertEquals(2, sentences.length);
+ assertEquals("„Halt!“", sentences[0]);
+ assertEquals("Er blieb stehen.", sentences[1]);
+ }
+
+ @Test
+ public void testSentSplitterGermanQuoteWithQuestion () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("„Wirklich?“ Sie nickte.");
+ assertEquals(2, sentences.length);
+ assertEquals("„Wirklich?“", sentences[0]);
+ assertEquals("Sie nickte.", sentences[1]);
+ }
+
+ @Test
+ public void testSentSplitterGermanQuoteWithEllipsis () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("„Na ja …“ Er schwieg.");
+ assertEquals(2, sentences.length);
+ assertEquals("„Na ja …“", sentences[0]);
+ assertEquals("Er schwieg.", sentences[1]);
+ }
+
+ @Test
+ public void testSentSplitterEllipsisWithoutQuote () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("Na ja … Er schwieg.");
+ assertEquals(2, sentences.length);
+ assertEquals("Na ja …", sentences[0]);
+ assertEquals("Er schwieg.", sentences[1]);
+ }
+
+ @Test
+ public void testSentSplitterEnglishClosingQuote () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("He said “Hello.” She left.");
+ assertEquals(2, sentences.length);
+ assertEquals("He said “Hello.”", sentences[0]);
+ assertEquals("She left.", sentences[1]);
+ }
+
+ @Test
+ public void testSentSplitterNoFalseQuoteAbsorption () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("Er sagte nichts. „Wirklich?“");
+ assertEquals(2, sentences.length);
+ assertEquals("Er sagte nichts.", sentences[0]);
+ assertEquals("„Wirklich?“", sentences[1]);
+ }
+
+ @Test
+ public void testSentSplitterMultipleGermanQuotedSentences () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("„Ja.“ „Nein.“ „Vielleicht.“");
+ assertEquals(3, sentences.length);
+ assertEquals("„Ja.“", sentences[0]);
+ assertEquals("„Nein.“", sentences[1]);
+ assertEquals("„Vielleicht.“", sentences[2]);
+ }
+
+ @Test
+ public void testSentSplitterGermanQuotePosDetect () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ Span[] spans = tok.sentPosDetect("„Halt!“ Er ging.");
+ assertEquals(2, spans.length);
+ assertEquals(0, spans[0].getStart());
+ assertEquals(7, spans[0].getEnd());
+ assertEquals("„Halt!“", spans[0].getType());
+ assertEquals(8, spans[1].getStart());
+ assertEquals(16, spans[1].getEnd());
+ assertEquals("Er ging.", spans[1].getType());
+ }
+
+ @Test
+ public void testSentSplitterGermanOpeningQuoteTokenized () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] tokens = tok.tokenize("„Hallo“");
+ assertEquals(3, tokens.length);
+ assertEquals("„", tokens[0]);
+ assertEquals("Hallo", tokens[1]);
+ assertEquals("“", tokens[2]);
+ }
+
+ @Test
+ public void testSentSplitterSimpleUnchanged () {
+ DerekoDfaTokenizer_de tok = new DerekoDfaTokenizer_de();
+ String[] sentences = tok.sentDetect("Erste. Zweite. Dritte.");
+ assertEquals(3, sentences.length);
+ assertEquals("Erste.", sentences[0]);
+ assertEquals("Zweite.", sentences[1]);
+ assertEquals("Dritte.", sentences[2]);
+ }
}