commit 7198645d17c76a8442ebcb306b84bfd702209ffd
author Akron <nils@diewald-online.de> Tue Nov 09 01:36:30 2021 +0100
committer Akron <nils@diewald-online.de> Tue Nov 09 01:36:30 2021 +0100
tree 36dd57e7d0a8e30e342113e43908d8ffe1f5fbfc
parent 6742b96b605b9a48a68e7c5b68da684de37e4f66

Speed up build

Change-Id: I15ca89cdd79f7291c8f77f0e9cd31c6d56db186f

src/tokenizer.xfst

diff --git a/src/tokenizer.xfst b/src/tokenizer.xfst
index dca20c4..30d06e8 100644
--- a/src/tokenizer.xfst
+++ b/src/tokenizer.xfst
@@ -225,7 +225,7 @@
 
 echo - Compile Real Token
 
-define RealToken [Punct|Word|XML|Email|URL|SNS|Domain|AcronymDep|File|Ord|Num|Years|Times];
+define RealToken [Punct|Word|SNS|AcronymDep|Ord|Num|Years|Times];
 
 echo - Introduce Token splitter
 
@@ -233,7 +233,12 @@
   XMLEntities @-> ... NLout,
   Abbr @-> ... NLout,
   RealToken @-> ... NLout,
+  XML @-> ... NLout,
+  URL @-> ... NLout,
+  Email @-> ... NLout,
+  File @-> ... NLout,
   Plusampersand @-> ... NLout,
+  Domain @-> ... NLout,
   Emoji @-> ... NLout,
   [Streetname|Omission|Emdash] @-> ... NLout
   ]