Introduce --[no]-tokens flag
Change-Id: I3aff53491151d8fe4e00d9f6747f8f12f6051a54
diff --git a/token_writer.go b/token_writer.go
index 32c5a99..5e4b72d 100644
--- a/token_writer.go
+++ b/token_writer.go
@@ -35,7 +35,7 @@
}
// Create a new token writer based on the options
-func NewTokenWriterFromOptions(w io.Writer, positionFlag bool, newlineAfterEot bool) *TokenWriter {
+func NewTokenWriterFromOptions(w io.Writer, positionFlag bool, tokenFlag bool, newlineAfterEot bool) *TokenWriter {
writer := bufio.NewWriter(w)
posC := 0
pos := make([]int, 0, 200)
@@ -59,8 +59,10 @@
posC += len(buf) - offset
pos = append(pos, posC)
- writer.WriteString(string(buf[offset:]))
- writer.WriteRune('\n')
+ if tokenFlag {
+ writer.WriteString(string(buf[offset:]))
+ writer.WriteRune('\n')
+ }
}
} else {
tw.Token = func(offset int, buf []rune) {