Cleanup
diff --git a/matrix.go b/matrix.go
index ac53b9a..b800d1a 100644
--- a/matrix.go
+++ b/matrix.go
@@ -268,9 +268,6 @@
sigmaCount := int(bo.Uint16(buf[12:14]))
arraySize := (mat.stateCount + 1) * sigmaCount
- // Shouldn't be relevant though
- // mat.maxSize = arraySize - 1
-
for x := 0; x < sigmaCount; x++ {
sym, _, err := r.ReadRune()
if err == nil && sym != 0 {
@@ -397,11 +394,6 @@
// Maybe not necessary - and should be simpler!
// Just Remove
t0 &= ^FIRSTBIT
- /*
- if (t0 & FIRSTBIT) != 0 {
- t0 ^= FIRSTBIT
- }
- */
epsilonState = t0
epsilonOffset = buffo
@@ -413,21 +405,13 @@
// Checks a transition based on t0, a and buffo
t = mat.array[(int(a)-1)*mat.stateCount+int(t0)]
- // t = mat.array[t0].getBase() + uint32(a)
- // ta := dat.array[t]
if DEBUG {
// Char is only relevant if set
fmt.Println("Check", t0, "-", a, "(", string(char), ")", "->", t)
- /*
- if false {
- fmt.Println(dat.outgoing(t0))
- }
- */
}
// Check if the transition is invalid according to the double array
- // if t > dat.array[1].getCheck() || ta.getCheck() != t0 {
if t == 0 {
if DEBUG {
@@ -472,7 +456,6 @@
buffo++
// Transition does not produce a character
- // if buffo == 1 && ta.isNonToken() {
if buffo == 1 && (t&FIRSTBIT) != 0 {
if DEBUG {
fmt.Println("Nontoken forward", showBuffer(buffer, buffo, buffi))
@@ -482,7 +465,6 @@
} else {
// Transition marks the end of a token - so flush the buffer
-
if buffi > 0 {
if DEBUG {
fmt.Println("-> Flush buffer: [", string(buffer[:buffo]), "]", showBuffer(buffer, buffo, buffi))
@@ -522,24 +504,6 @@
}
}
- // Move to representative state
- /*
- if ta.isSeparate() {
- t = ta.getBase()
- ta = dat.array[t]
-
- if DEBUG {
- fmt.Println("Representative pointing to", t)
- }
- }
- */
-
- // Ignore nontoken mark
- /*
- if t < 0 {
- t *= -1
- }
- */
t &= ^FIRSTBIT
newchar = true
@@ -559,50 +523,12 @@
if DEBUG {
fmt.Println("Entering final check")
}
- /*
- // Automaton is in a final state, so flush the buffer and return
- x := dat.array[t].getBase() + uint32(dat.final)
-
- if x < dat.array[1].getCheck() && dat.array[x].getCheck() == t {
-
- if buffi > 0 {
- if DEBUG {
- fmt.Println("-> Flush buffer: [", string(buffer[:buffi]), "]")
- }
- writer.WriteString(string(buffer[:buffi]))
-
- if dat.array[t].isTokenEnd() {
- writer.WriteRune('\n')
- if DEBUG {
- fmt.Println("-> Newline")
- }
- }
- }
-
- // Add an additional sentence ending, if the file is over but no explicit
- // sentence split was reached. This may be controversial and therefore
- // optional via parameter.
- if !dat.array[t0].isTokenEnd() {
- writer.WriteRune('\n')
- if DEBUG {
- fmt.Println("-> Newline")
- }
- }
-
- // TODO:
- // There may be a new line at the end, from an epsilon,
- // so we may need to go on!
- return true
- }
- */
// Check epsilon transitions until a final state is reached
t0 = t
- // t = dat.array[t0].getBase() + uint32(dat.epsilon)
t = mat.array[(int(mat.epsilon)-1)*mat.stateCount+int(t0)]
a = mat.epsilon
newchar = false
- // if dat.array[t].getCheck() == t0 {
// t can't be < 0
if t != 0 {
// Remember state for backtracking to last tokenend state