| commit | b71b51fe1e63950dc84d18a3face5c5eefc5915f | [log] [tgz] |
|---|---|---|
| author | Akron <nils@diewald-online.de> | Mon Mar 04 15:39:02 2024 +0100 |
| committer | Akron <nils@diewald-online.de> | Mon Mar 04 15:39:02 2024 +0100 |
| tree | 0791bf0c2fc829dfa85817d861a908a41bfec273 | |
| parent | 0b9aa666d6e824452935c84e8f88637f5017499c [diff] |
Add comment to tokenwriter Change-Id: Ib00a0f15a193fc84d9a87dd0939d49362b5a8d62
diff --git a/token_writer.go b/token_writer.go index f6dffa4..9c09506 100644 --- a/token_writer.go +++ b/token_writer.go
@@ -8,6 +8,12 @@ type Bits uint8 +// TODO-Perf: +// - TokenWriter may support AvailableBuffer(), so tokens can be written +// directly without a separate buffer. copying from the same underlying +// byte array is a nop thren (Go 1.18). + + const ( TOKENS Bits = 1 << iota SENTENCES