commit e96895fdc402ab3591949da8a26dfc7b3c11bdc6
author Akron <nils@diewald-online.de> Tue Mar 08 19:58:37 2022 +0100
committer Akron <nils@diewald-online.de> Tue Mar 08 19:58:37 2022 +0100
tree e8f62f6c1d9437cdad3b0eb356cb79bf81f492b7
parent b02ad07215f8811b2a4c72b1725e0ebdd01ba03d

Improve handling of sentence splits including speech

Change-Id: Id699624f17e3e983b0900d48e95c1d7e782e2215

src/tokenizer.xfst

diff --git a/src/tokenizer.xfst b/src/tokenizer.xfst
index 8026c01..3d63e5b 100644
--- a/src/tokenizer.xfst
+++ b/src/tokenizer.xfst
@@ -44,15 +44,11 @@
            [%, %,]];
 
 ! Right punctuation - excluding the characters that can be used as apostrophe
-define RP [SP|","|";"|":"|
+define RPS ["”"|"›"|"»"|%"|[%’ %’]|["'" "'"]|[%‘ %‘]];
+define RP [SP|RPS|","|";"|":"|
               ")"|"]"|"}"|
-              "”"|"›"|"»"|
-              %"|
               ! differs
-              ["'" "'"]|
-              "*"|"/"|"_"| ! Can be Markdown
-              ! from book
-              [%‘ %‘]|[%’ %’]];
+              "*"|"/"|"_"]; ! Can be Markdown
 
 define Sym ["-"|"+"|"<"|">"|"*"|"/"|%=|%@|%&];
 define Apos %'|%’|%`;
@@ -221,9 +217,16 @@
   File @-> ... NLout,
   Domain @-> ... NLout,
   Emoji @-> ... NLout
-] .o. [[WS|NL]+ @-> 0 || [ .#. | NLout ] _ ];
+];
 
 echo - Introduce Sentence splitter
-read regex Token .o. [[["."|"!"|"?"]+|"…"] @-> ... NLout \/ NLout _ ];
+! And compose Whitespace ignorance
+read regex Token .o. [
+  SP NLout %" @-> ... NLout \/ _ NLout \%,
+] .o. [
+  SP @-> ... NLout \/ NLout _ NLout \%"
+] .o. [
+  [WS|NL]+ @-> 0 || [ .#. | NLout ] _
+];
 
 ! foma -e "source tokenizer.xfst" -q -s && cat text.txt | flookup tokenizer.fst -x -b
\ No newline at end of file