Normalise free-text creatDate to ISO for Krill type:date fields
Krill type:date fields expect an ISO date (YYYY[-MM[-DD]]), but creatDate
arrives in several free-text shapes. The previous naive '.'->'-' replacement
only worked for the canonical year-first form (1932.03.03) and mangled the
day-first forms found e.g. in RPK, optionally preceded by a weekday name:
"Donnerstag, 04.10.2018" became "Donnerstag, 04-10-2018".
Add normalizeKrillDate(), which extracts an ISO date from such values:
day-first DD.MM.YYYY (with optional leading weekday/comma) and year-first
YYYY.MM.DD both become YYYY-MM-DD; already-ISO values and the partial
YYYY.MM / YYYY forms pass through; for a date range the first date wins.
Wired into both creatDate -> creationDate and the plain <date> -> pubDate
fallback; the structured <pubDate type=year|month|day> path is untouched.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Change-Id: Ie546909f1a17e78d12a3139c8120bff12e1ea3a6
diff --git a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
index 739f8c7..afab174 100644
--- a/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
+++ b/app/src/main/kotlin/de/ids_mannheim/korapxmltools/KorapXmlTool.kt
@@ -5658,8 +5658,8 @@
metadata["textClass"] = finalTopics
}
- headerRoot.firstText("creatDate")?.replace(".", "-")?.let {
- metadata["creationDate"] = it
+ headerRoot.firstText("creatDate")?.let { raw ->
+ metadata["creationDate"] = normalizeKrillDate(raw) ?: raw.replace(".", "-")
}
var year: String? = null
@@ -5689,7 +5689,7 @@
val dateVal = dateEl?.getAttribute("when")?.trim()?.takeIf { it.isNotEmpty() }
?: dateEl?.textContent?.trim()?.takeIf { it.isNotEmpty() }
if (dateVal != null) {
- metadata["pubDate"] = dateVal
+ metadata["pubDate"] = normalizeKrillDate(dateVal) ?: dateVal
}
}
@@ -5764,6 +5764,35 @@
}
}
+ // Normalise a free-text I5 date (e.g. <creatDate> or a plain <date>) to an ISO
+ // date string (YYYY, YYYY-MM or YYYY-MM-DD), which is what Krill expects for
+ // type:date fields. Handles the canonical year-first dotted form (1932.03.03)
+ // as well as the day-first forms found in some corpora (RPK), optionally
+ // preceded by a weekday name and/or comma ("Donnerstag, 04.10.2018",
+ // "Donnerstag 05.01.2017"). Already-ISO values pass through unchanged. For a
+ // date range the first date wins. Returns null when no date is recognisable.
+ private fun normalizeKrillDate(raw: String?): String? {
+ val s = raw?.trim()?.replace(Regex("\\s+"), " ")?.takeIf { it.isNotEmpty() } ?: return null
+ fun pad(x: String) = x.padStart(2, '0')
+ // Full dates first (most specific), then year-month, then year only.
+ Regex("""\b(\d{1,2})\.(\d{1,2})\.(\d{4})\b""").find(s)?.let { // dd.mm.yyyy
+ val (d, m, y) = it.destructured
+ return "$y-${pad(m)}-${pad(d)}"
+ }
+ Regex("""\b(\d{4})\.(\d{1,2})\.(\d{1,2})\b""").find(s)?.let { // yyyy.mm.dd
+ val (y, m, d) = it.destructured
+ return "$y-${pad(m)}-${pad(d)}"
+ }
+ Regex("""\b\d{4}-\d{2}-\d{2}\b""").find(s)?.let { return it.value } // yyyy-mm-dd (ISO)
+ Regex("""\b(\d{4})\.(\d{1,2})\b""").find(s)?.let { // yyyy.mm
+ val (y, m) = it.destructured
+ return "$y-${pad(m)}"
+ }
+ Regex("""\b\d{4}-\d{2}\b""").find(s)?.let { return it.value } // yyyy-mm (ISO)
+ Regex("""\b\d{4}\b""").find(s)?.let { return it.value } // yyyy
+ return null
+ }
+
private fun composeKrillPubDate(
year: String?,
month: String?,
diff --git a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
index aec8a92..bf78797 100644
--- a/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
+++ b/app/src/test/kotlin/de/ids_mannheim/korapxmltools/KrillJsonGeneratorTest.kt
@@ -548,6 +548,53 @@
}
@Test
+ fun creatDateIsNormalisedToIsoForVariousI5Forms() {
+ // creatDate appears in several free-text shapes across corpora. Each must
+ // reach the Krill type:date field as an ISO date. The day-first weekday
+ // forms come from the RPK corpus; the year-first dotted form is canonical.
+ val cases = mapOf(
+ "1932.03.03" to "1932-03-03",
+ "Donnerstag, 04.10.2018" to "2018-10-04",
+ "Donnerstag 05.01.2017" to "2017-01-05",
+ "2023.03.09" to "2023-03-09",
+ "1984-01-02" to "1984-01-02",
+ // Month-only and year-only values keep their (reduced) ISO precision.
+ "2020.05" to "2020-05",
+ "2011.02" to "2011-02",
+ "2016" to "2016"
+ )
+ cases.forEach { (input, expected) ->
+ val metadata = collectKrillMetadata(
+ KorapXmlTool(),
+ "TEST_DOC.1",
+ headerElement("<idsHeader><creatDate>$input</creatDate></idsHeader>")
+ )
+ assertEquals(expected, metadata["creationDate"], "creatDate '$input' should normalise to ISO")
+ // creationDate backfills pubDate, so the document carries a clean date pair.
+ assertEquals(expected, metadata["pubDate"], "pubDate should backfill from creatDate '$input'")
+ }
+ }
+
+ @Test
+ fun creatDatePeriodsCollapseToASingleEndpointDate() {
+ // A creatDate is sometimes a period. Krill needs a single date, so we keep
+ // one endpoint (the first when the dates are uniformly formatted). Whichever
+ // endpoint is kept, it must be a valid ISO date from the period.
+ fun creationDateFor(input: String): String? = collectKrillMetadata(
+ KorapXmlTool(),
+ "TEST_DOC.1",
+ headerElement("<idsHeader><creatDate>$input</creatDate></idsHeader>")
+ )["creationDate"] as? String
+
+ // Uniformly dotted periods keep the first date.
+ assertEquals("1920-05-01", creationDateFor("1920.05.01-1923.05.02"))
+ assertEquals("2020-05", creationDateFor("2020.05-2021.06"))
+
+ // A mixed-separator period still collapses to a valid endpoint date.
+ assertContains(setOf("1920-05-01", "1923-05-02"), creationDateFor("1920.05-01-1923.05.02"))
+ }
+
+ @Test
fun krillInheritedDatesIgnoreEmptyTextValuesAndBackfillEachOther() {
val tool = KorapXmlTool()
val textMetadata = collectKrillMetadata(