Minor improvements
Change-Id: I917c2def6174ce649bcb5fff47a31cd821c90831
diff --git a/src/tokenizer.xfst b/src/tokenizer.xfst
index 9b9f663..6b2331e 100644
--- a/src/tokenizer.xfst
+++ b/src/tokenizer.xfst
@@ -209,12 +209,11 @@
echo - Compile Real Token
-define RealToken [Punct|Emdash|Word|SNS|AcronymDep|Ord|Num|Years|Times|XMLEntities|Omission];
+define RealToken [Punct|Emdash|Abbr|Streetname|Word|SNS|AcronymDep|Ord|Num|Years|Times|XMLEntities|Omission];
echo - Introduce Token splitter
define Token [
- [Abbr|Streetname] @-> ... NLout,
RealToken @-> ... NLout,
XML @-> ... NLout,
URL @-> ... NLout,
diff --git a/token_writer.go b/token_writer.go
index 250d536..ed580ef 100644
--- a/token_writer.go
+++ b/token_writer.go
@@ -54,7 +54,7 @@
// and write to string
// Accept newline after EOT
- if flags&NEWLINE_AFTER_EOT != 0 && posC == 0 && buf[0] == '\n' && !init {
+ if posC == 0 && flags&NEWLINE_AFTER_EOT != 0 && buf[0] == '\n' && !init {
posC--
}
@@ -152,7 +152,7 @@
// Collect text ends
} else {
tw.TextEnd = func(_ int) {
- writer.WriteRune('\n')
+ writer.WriteByte('\n')
writer.Flush()
}
}