commit e87906bb377ede4f8174bc5336da00e80bae9662
author Akron <nils@diewald-online.de> Wed Nov 24 10:39:14 2021 +0100
committer Akron <nils@diewald-online.de> Wed Nov 24 10:39:14 2021 +0100
tree 796919c0aec64e99046dcb14b955c2c7ba2b3e26
parent 90aa45b783e6af64cbe76da0c4441228117d4ea2

Minor improvements

Change-Id: I917c2def6174ce649bcb5fff47a31cd821c90831

src/tokenizer.xfst

diff --git a/src/tokenizer.xfst b/src/tokenizer.xfst
index 9b9f663..6b2331e 100644
--- a/src/tokenizer.xfst
+++ b/src/tokenizer.xfst
@@ -209,12 +209,11 @@
 
 echo - Compile Real Token
 
-define RealToken [Punct|Emdash|Word|SNS|AcronymDep|Ord|Num|Years|Times|XMLEntities|Omission];
+define RealToken [Punct|Emdash|Abbr|Streetname|Word|SNS|AcronymDep|Ord|Num|Years|Times|XMLEntities|Omission];
 
 echo - Introduce Token splitter
 
 define Token [
-  [Abbr|Streetname] @-> ... NLout,
   RealToken @-> ... NLout,
   XML @-> ... NLout,
   URL @-> ... NLout,