Ignore tokenend accepting transitions
diff --git a/datokenizer.go b/datokenizer.go
index 8fb3d19..f455ec3 100644
--- a/datokenizer.go
+++ b/datokenizer.go
@@ -348,6 +348,9 @@
")")
return nil
}
+ } else if inSym == tok.tokenend {
+ // Ignore tokenend accepting arcs
+ continue
} else if inSym == tok.epsilon {
log.Println("General epsilon transitions are not supported")
return nil
diff --git a/datokenizer_test.go b/datokenizer_test.go
index 8313c1e..b6383d4 100644
--- a/datokenizer_test.go
+++ b/datokenizer_test.go
@@ -120,7 +120,7 @@
buf := bytes.NewBuffer(b)
n, err := dat.WriteTo(buf)
assert.Nil(err)
- assert.Equal(int64(224), n)
+ assert.Equal(int64(208), n)
dat2 := ParseDatok(buf)
assert.NotNil(dat2)
@@ -135,8 +135,8 @@
assert.True(tmatch(dat2, "bad"))
assert.True(tmatch(dat2, "wald gehen"))
- assert.Equal(dat.TransCount(), 19)
- assert.Equal(dat2.TransCount(), 19)
+ assert.Equal(dat.TransCount(), 17)
+ assert.Equal(dat2.TransCount(), 17)
}
func TestIgnorableMCS(t *testing.T) {
@@ -159,7 +159,7 @@
assert.Equal("b", tokens[1])
assert.Equal("<ab>", tokens[2])
assert.Equal(4, len(tokens))
- assert.Equal(dat.TransCount(), 17)
+ assert.Equal(dat.TransCount(), 15)
}
func TestFullTokenizer(t *testing.T) {