Extend tokenizer fileending
Change-Id: Ic9d40178daec364793cad52dd2e0560a40479eb9
diff --git a/src/tokenizer.xfst b/src/tokenizer.xfst
index 0100d6e..742f147 100644
--- a/src/tokenizer.xfst
+++ b/src/tokenizer.xfst
@@ -16,6 +16,7 @@
define Digit [%0|1|2|3|4|5|6|7|8|9];
define AsciiLetter [a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z];
+define HexLetter [Digit|a|A|b|B|c|C|d|D|e|E|f|F];
define EOT "\u0004";
!!!!!!!!!!!!!!!!!
@@ -110,6 +111,7 @@
define Letter [ [ AsciiLetter | ö | ü | ä | ß ] .o. Caseinsensitive ];
+! Abbreviations and Initials
define Abbr [ @txt"txt/abbrv.txt" | Letter ] %.;
define Plusampersand @txt"txt/plusampersand.txt";
@@ -175,6 +177,7 @@
["m" "p" ["3"|"4"]]|
{ogg}|
{png}|
+ [{ppt} ("x")]|
{avi}|
{txt}|
{xls}|