Ignore MCS in sigma if not used in the transducer
diff --git a/datokenizer.go b/datokenizer.go
index 9f8079b..805b8df 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -74,6 +74,7 @@
// of the tokenizer.
type Tokenizer struct {
sigmaRev map[int]rune
+ sigmaMCS map[int]string
arcCount int
sigmaCount int
transitions []map[int]*edge
@@ -137,6 +138,7 @@
tok := &Tokenizer{
sigmaRev: make(map[int]rune),
+ sigmaMCS: make(map[int]string),
epsilon: -1,
unknown: -1,
identity: -1,
@@ -352,10 +354,13 @@
")")
return nil
}
-
} else if inSym == tok.epsilon {
log.Println("General epsilon transitions are not supported")
return nil
+ } else if tok.sigmaMCS[inSym] != "" {
+ log.Fatalln("Non supported character", tok.sigmaMCS[inSym])
+ } else if tok.sigmaMCS[outSym] != "" {
+ log.Fatalln("Non supported character", tok.sigmaMCS[outSym])
}
// Create an edge based on the collected information
@@ -451,8 +456,8 @@
}
default:
{
- log.Println("MCS not supported: " + line)
- return nil
+ // MCS not supported
+ tok.sigmaMCS[number] = line
}
}
continue
@@ -464,8 +469,9 @@
return nil
}
if len(line) != 1 {
- log.Println("MCS not supported:" + line)
- return nil
+ // MCS not supported
+ tok.sigmaMCS[number] = line
+ continue
}
symbol = rune('\n')
}
@@ -474,7 +480,7 @@
}
}
}
-
+ tok.sigmaMCS = nil
return tok
}