Support both matrix and da in the command
diff --git a/fomafile.go b/fomafile.go
index 1d46500..2b10736 100644
--- a/fomafile.go
+++ b/fomafile.go
@@ -29,6 +29,7 @@
type Tokenizer interface {
Transduce(r io.Reader, w io.Writer) bool
+ Type() string
}
// Automaton is the intermediate representation
@@ -442,6 +443,40 @@
return auto
}
+func LoadTokenizerFile(file string) Tokenizer {
+ f, err := os.Open(file)
+ if err != nil {
+ log.Println(err)
+ return nil
+ }
+ defer f.Close()
+
+ gz, err := gzip.NewReader(f)
+ if err != nil {
+ log.Println(err)
+ return nil
+ }
+ defer gz.Close()
+
+ r := bufio.NewReader(gz)
+
+ mstr, err := r.Peek(len(DAMAGIC))
+
+ if err != nil {
+ log.Println(err)
+ return nil
+ }
+
+ if string(mstr) == MAMAGIC {
+ return ParseMatrix(r)
+ } else if string(mstr) == DAMAGIC {
+ return ParseDatok(r)
+ }
+
+ log.Println("Neither a matrix nor a datok file")
+ return nil
+}
+
// Set alphabet A to the list of all symbols
// outgoing from s
func (auto *Automaton) getSet(s int, A *[]int) {