Fix buffer bug in token writer
Change-Id: I615618fe7833e1b97ae86b23d1fee760401154db
diff --git a/testdata/clitic_test.fst b/testdata/clitic_test.fst
new file mode 100644
index 0000000..b373352
--- /dev/null
+++ b/testdata/clitic_test.fst
Binary files differ
diff --git a/testdata/clitic_test.matok b/testdata/clitic_test.matok
new file mode 100644
index 0000000..9cc4971
--- /dev/null
+++ b/testdata/clitic_test.matok
Binary files differ
diff --git a/testdata/clitic_test.xfst b/testdata/clitic_test.xfst
new file mode 100644
index 0000000..1b129aa
--- /dev/null
+++ b/testdata/clitic_test.xfst
@@ -0,0 +1,13 @@
+define TB "@_TOKEN_BOUND_@";
+define WS [" "|"\u000a"|"\u0009"];
+define PUNCT ["."|"?"|"!"];
+define Char \[WS|PUNCT];
+define Word Char+;
+
+! Compose token boundaries
+define Tokenizer [[Word|PUNCT] @-> ... TB] .o.
+ ! Compose Whitespace ignorance
+[WS+ @-> 0] .o.
+ ! Compose sentence ends
+[[PUNCT+] @-> ... TB \/ TB _ ];
+read regex Tokenizer .o. [{'re} ->@ TB ... ];