Introduced command line tool
diff --git a/cmd/datok.go b/cmd/datok.go
new file mode 100644
index 0000000..1d3bbbc
--- /dev/null
+++ b/cmd/datok.go
@@ -0,0 +1,44 @@
+package main
+
+import (
+ "os"
+
+ datok "github.com/KorAP/datokenizer"
+ "github.com/alecthomas/kong"
+)
+
+var cli struct {
+ Tokenizer string `kong:"required,short='t',help='The Double Array Tokenizer file'"`
+}
+
+// Main method for command line handling
+func main() {
+
+ // Parse command line parameters
+ parser := kong.Must(
+ &cli,
+ kong.Name("datok"),
+ kong.Description("Double Array based tokenizer"),
+ kong.UsageOnError(),
+ )
+
+ _, err := parser.Parse(os.Args[1:])
+
+ parser.FatalIfErrorf(err)
+
+ // Load the Datok file
+ dat := datok.LoadDatokFile(cli.Tokenizer)
+
+ // Unable to load the datok file
+ if dat == nil {
+ os.Exit(1)
+ }
+
+ // Program is running in a pipe
+ fileInfo, _ := os.Stdin.Stat()
+ if fileInfo.Mode()&os.ModeCharDevice == 0 {
+
+ // Transduce from STDIN and write to STDOUT
+ dat.Transduce(os.Stdin, os.Stdout)
+ }
+}
diff --git a/datokenizer.go b/datokenizer.go
index f2d9304..270b61e 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -14,14 +14,16 @@
// Serialization is little endian.
// TODO:
+// - Write simple main function.
+// - Turn sigma into an array instead of using a map.
// - replace maxSize with the check value
// - Add checksum to serialization.
-// - Mark epsilon transitions in bytes
// - Introduce methods on BC array entries instead of
// jumping into the entries all the time!
// - Instead of memoizing the loadFactor, better remember
// the number of set transitions
// - Replace table with a map
+// - Mark epsilon transitions in bytes
import (
"bufio"
diff --git a/go.mod b/go.mod
index 9d677ca..6104f40 100644
--- a/go.mod
+++ b/go.mod
@@ -3,6 +3,7 @@
go 1.16
require (
+ github.com/alecthomas/kong v0.2.17
github.com/rs/zerolog v1.23.0 // indirect
github.com/stretchr/testify v1.7.0
)
diff --git a/go.sum b/go.sum
index e4d7005..8942bc1 100644
--- a/go.sum
+++ b/go.sum
@@ -1,7 +1,12 @@
+github.com/alecthomas/kong v0.2.17 h1:URDISCI96MIgcIlQyoCAlhOmrSw6pZScBNkctg8r0W0=
+github.com/alecthomas/kong v0.2.17/go.mod h1:ka3VZ8GZNPXv9Ov+j4YNLkI8mTuhXyr/0ktSlqIydQQ=
github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@@ -37,3 +42,5 @@
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=