Add XML entities
Change-Id: Ib22e51ec8427e0af23a9dcf83c01e6e16837c91e
diff --git a/src/tokenizer.xfst b/src/tokenizer.xfst
index 742f147..dca20c4 100644
--- a/src/tokenizer.xfst
+++ b/src/tokenizer.xfst
@@ -54,7 +54,7 @@
! from book
[%‘ %‘]|[%’ %’]];
-define Sym ["-"|"+"|"<"|">"|"*"|"/"|%=|%@];
+define Sym ["-"|"+"|"<"|">"|"*"|"/"|%=|%@|%&];
define Apos %'|%’|%`;
define Punct [LP|RP|Sym];
!define nonSym \[WS|LP|RP|Sym];
@@ -163,7 +163,12 @@
] WS* ">"
].u;
-!define Email [Alpha [URLChar-At]* At Alpha URLChar* [Dot [[Alpha URLChar+]-Dot-At]]+];
+! XML entities
+source entities.xfst
+define XMLEntities;
+
+
+! Email addresses
define Email URLChar+ At URLChar+ [Dot URLChar+]+;
! Twitter user, hashtag, Google+
@@ -225,6 +230,7 @@
echo - Introduce Token splitter
define Token [
+ XMLEntities @-> ... NLout,
Abbr @-> ... NLout,
RealToken @-> ... NLout,
Plusampersand @-> ... NLout,