commit cae39115eefcf3d8b813c62a367e89ea8251ad79
author Akron <nils@diewald-online.de> Wed Apr 26 19:43:16 2023 +0200
committer Akron <nils@diewald-online.de> Wed Apr 26 19:43:16 2023 +0200
tree 33e87a30fcacf442dc9312cf3a0ccf47b7ec38c1
parent d0dfea8258ea50be202a9f500e76a9285c8ad01e

Fix buffer bug in token writer

Change-Id: I615618fe7833e1b97ae86b23d1fee760401154db

testdata/clitic_test.xfst

diff --git a/testdata/clitic_test.xfst b/testdata/clitic_test.xfst
new file mode 100644
index 0000000..1b129aa
--- /dev/null
+++ b/testdata/clitic_test.xfst
@@ -0,0 +1,13 @@
+define TB "@_TOKEN_BOUND_@";
+define WS [" "|"\u000a"|"\u0009"];
+define PUNCT ["."|"?"|"!"];
+define Char \[WS|PUNCT];
+define Word Char+;
+
+! Compose token boundaries
+define Tokenizer [[Word|PUNCT] @-> ... TB] .o.
+ ! Compose Whitespace ignorance
+[WS+ @-> 0] .o.
+ ! Compose sentence ends
+[[PUNCT+] @-> ... TB \/ TB _ ];
+read regex Tokenizer .o. [{'re} ->@ TB ... ];