commit 6dcb6ce65c603457c9855d79614fe5b4a844bcfc
author Akron <nils@diewald-online.de> Sat Apr 09 16:09:51 2022 +0200
committer Akron <nils@diewald-online.de> Sat Apr 09 16:09:51 2022 +0200
tree 86b445b972444305b824c391015d18f5a9a6c732
parent 3b6c7fb9a47a476e982cd963e776a1715bb42704

Add arrows

Change-Id: I250a30cafacb5ddfd7c87d5c11ab7c0ead923dd0

src/all/allpost.xfst

diff --git a/src/all/allpost.xfst b/src/all/allpost.xfst
index 6ea6dcf..32e7a90 100644
--- a/src/all/allpost.xfst
+++ b/src/all/allpost.xfst
@@ -16,6 +16,9 @@
 ! XML entities
 source all/entities.xfst
 
+! Arrows
+define Arrows [Alldash ">" | "<" Alldash];
+
 ! Technical protocols
 source all/protocols.xfst
 

src/all/allpref.xfst

diff --git a/src/all/allpref.xfst b/src/all/allpref.xfst
index bb5183f..7175d0b 100644
--- a/src/all/allpref.xfst
+++ b/src/all/allpref.xfst
@@ -58,6 +58,7 @@
 
 define Emdash [%- %- (%-)+ | ["\u2014"|"\u2015"|"\u2e3a"|"\u2e3b"|"\ufe58"]+];
 define Dash ["-"|"\u2011"|"\u2012"|"\u2013"|"\u2e1a"|"\ufe63"|"\uff0d"];
+define Alldash ["-"|"\u2011"|"\u2012"|"\u2013"|"\u2e1a"|"\ufe63"|"\uff0d"|"\u2014"|"\u2015"|"\u2e3a"|"\u2e3b"|"\ufe58"];
 define Slash ["⁄"|"∕"|"／"|"/"];
 define Asterisk ["*"];
 

src/de/tokenizer.xfst

diff --git a/src/de/tokenizer.xfst b/src/de/tokenizer.xfst
index 407c482..9670f9a 100644
--- a/src/de/tokenizer.xfst
+++ b/src/de/tokenizer.xfst
@@ -73,7 +73,7 @@
   Email @-> ... NLout,
   File @-> ... NLout,
   Domain @-> ... NLout,
-  Emoticons @-> ... NLout
+  [Emoticons|Arrows] @-> ... NLout
 ];
 
 source all/allsentencesplit.xfst