Add support keeping rewrites
diff --git a/ast/ast.go b/ast/ast.go
index 685483c..07a6df1 100644
--- a/ast/ast.go
+++ b/ast/ast.go
@@ -19,6 +19,7 @@
TokenNode NodeType = "token"
TermGroupNode NodeType = "termGroup"
TermNode NodeType = "term"
+ RewriteNode NodeType = "rewrite"
AndRelation RelationType = "and"
OrRelation RelationType = "or"
MatchEqual MatchType = "eq"
@@ -30,9 +31,23 @@
Type() NodeType
}
+// Rewrite represents a koral:rewrite
+type Rewrite struct {
+ Editor string `json:"editor,omitempty"`
+ Operation string `json:"operation,omitempty"`
+ Scope string `json:"scope,omitempty"`
+ Src string `json:"src,omitempty"`
+ Comment string `json:"_comment,omitempty"`
+}
+
+func (r *Rewrite) Type() NodeType {
+ return RewriteNode
+}
+
// Token represents a koral:token
type Token struct {
- Wrap Node `json:"wrap"`
+ Wrap Node `json:"wrap"`
+ Rewrites []Rewrite `json:"rewrites,omitempty"`
}
func (t *Token) Type() NodeType {
@@ -43,6 +58,7 @@
type TermGroup struct {
Operands []Node `json:"operands"`
Relation RelationType `json:"relation"`
+ Rewrites []Rewrite `json:"rewrites,omitempty"`
}
func (tg *TermGroup) Type() NodeType {
@@ -51,11 +67,12 @@
// Term represents a koral:term
type Term struct {
- Foundry string `json:"foundry"`
- Key string `json:"key"`
- Layer string `json:"layer"`
- Match MatchType `json:"match"`
- Value string `json:"value,omitempty"`
+ Foundry string `json:"foundry"`
+ Key string `json:"key"`
+ Layer string `json:"layer"`
+ Match MatchType `json:"match"`
+ Value string `json:"value,omitempty"`
+ Rewrites []Rewrite `json:"rewrites,omitempty"`
}
func (t *Term) Type() NodeType {
diff --git a/ast/ast_test.go b/ast/ast_test.go
index 1311a46..0daaf7c 100644
--- a/ast/ast_test.go
+++ b/ast/ast_test.go
@@ -36,6 +36,16 @@
},
expected: TermNode,
},
+ {
+ name: "Rewrite node returns correct type",
+ node: &Rewrite{
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ },
+ expected: RewriteNode,
+ },
}
for _, tt := range tests {
@@ -61,14 +71,26 @@
Value: "Pdt",
}
+ rewrites := []Rewrite{
+ {
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ Comment: "Default foundry has been added.",
+ },
+ }
+
group := &TermGroup{
Operands: []Node{term1, term2},
Relation: AndRelation,
+ Rewrites: rewrites,
}
assert.Len(t, group.Operands, 2)
assert.Equal(t, AndRelation, group.Relation)
assert.Equal(t, TermGroupNode, group.Type())
+ assert.Equal(t, rewrites, group.Rewrites)
// Test operands are correctly set
assert.Equal(t, term1, group.Operands[0])
@@ -83,10 +105,24 @@
Match: MatchEqual,
}
- token := &Token{Wrap: term}
+ rewrites := []Rewrite{
+ {
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ Comment: "Default foundry has been added.",
+ },
+ }
+
+ token := &Token{
+ Wrap: term,
+ Rewrites: rewrites,
+ }
assert.Equal(t, TokenNode, token.Type())
assert.Equal(t, term, token.Wrap)
+ assert.Equal(t, rewrites, token.Rewrites)
}
func TestTermConstruction(t *testing.T) {
@@ -99,6 +135,7 @@
match MatchType
hasValue bool
value string
+ rewrites []Rewrite
}{
{
name: "Term without value",
@@ -144,6 +181,38 @@
match: MatchNotEqual,
hasValue: false,
},
+ {
+ name: "Term with rewrites",
+ term: &Term{
+ Foundry: "opennlp",
+ Key: "DET",
+ Layer: "p",
+ Match: MatchEqual,
+ Rewrites: []Rewrite{
+ {
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ Comment: "Default foundry has been added.",
+ },
+ },
+ },
+ foundry: "opennlp",
+ key: "DET",
+ layer: "p",
+ match: MatchEqual,
+ hasValue: false,
+ rewrites: []Rewrite{
+ {
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ Comment: "Default foundry has been added.",
+ },
+ },
+ },
}
for _, tt := range tests {
@@ -158,6 +227,11 @@
} else {
assert.Empty(t, tt.term.Value)
}
+ if tt.rewrites != nil {
+ assert.Equal(t, tt.rewrites, tt.term.Rewrites)
+ } else {
+ assert.Empty(t, tt.term.Rewrites)
+ }
})
}
}
@@ -251,6 +325,23 @@
}
}
+func TestRewriteConstruction(t *testing.T) {
+ rewrite := &Rewrite{
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ Comment: "Default foundry has been added.",
+ }
+
+ assert.Equal(t, RewriteNode, rewrite.Type())
+ assert.Equal(t, "Kustvakt", rewrite.Editor)
+ assert.Equal(t, "operation:injection", rewrite.Operation)
+ assert.Equal(t, "foundry", rewrite.Scope)
+ assert.Equal(t, "Kustvakt", rewrite.Src)
+ assert.Equal(t, "Default foundry has been added.", rewrite.Comment)
+}
+
func TestComplexNestedStructures(t *testing.T) {
// Create a complex nested structure
innerGroup1 := &TermGroup{
diff --git a/parser/parser.go b/parser/parser.go
index e9b1102..6c3c750 100644
--- a/parser/parser.go
+++ b/parser/parser.go
@@ -24,6 +24,7 @@
Layer string `json:"layer,omitempty"`
Match string `json:"match,omitempty"`
Value string `json:"value,omitempty"`
+ Rewrites []ast.Rewrite `json:"rewrites,omitempty"`
// Store any additional fields
Extra map[string]any `json:"-"`
}
@@ -48,7 +49,7 @@
r.Extra = make(map[string]any)
for k, v := range raw {
switch k {
- case "@type", "wrap", "operands", "relation", "foundry", "key", "layer", "match", "value":
+ case "@type", "wrap", "operands", "relation", "foundry", "key", "layer", "match", "value", "rewrites":
continue
default:
r.Extra[k] = v
@@ -89,6 +90,9 @@
if r.Value != "" {
raw["value"] = r.Value
}
+ if len(r.Rewrites) > 0 {
+ raw["rewrites"] = r.Rewrites
+ }
// Add any extra fields
maps.Copy(raw, r.Extra)
@@ -123,7 +127,7 @@
if err != nil {
return nil, fmt.Errorf("error parsing wrapped node: %w", err)
}
- return &ast.Token{Wrap: wrap}, nil
+ return &ast.Token{Wrap: wrap, Rewrites: raw.Rewrites}, nil
case "koral:termGroup":
if len(raw.Operands) == 0 {
@@ -153,6 +157,7 @@
return &ast.TermGroup{
Operands: operands,
Relation: relation,
+ Rewrites: raw.Rewrites,
}, nil
case "koral:term":
@@ -170,11 +175,12 @@
}
return &ast.Term{
- Foundry: raw.Foundry,
- Key: raw.Key,
- Layer: raw.Layer,
- Match: match,
- Value: raw.Value,
+ Foundry: raw.Foundry,
+ Key: raw.Key,
+ Layer: raw.Layer,
+ Match: match,
+ Value: raw.Value,
+ Rewrites: raw.Rewrites,
}, nil
default:
diff --git a/parser/parser_test.go b/parser/parser_test.go
index 87405e4..25b2ad1 100644
--- a/parser/parser_test.go
+++ b/parser/parser_test.go
@@ -293,6 +293,145 @@
wantErr: false,
},
{
+ name: "Parse token with term and rewrites",
+ input: `{
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "Baum",
+ "layer": "orth",
+ "match": "match:eq",
+ "rewrites": [
+ {
+ "@type": "koral:rewrite",
+ "_comment": "Default foundry has been added.",
+ "editor": "Kustvakt",
+ "operation": "operation:injection",
+ "scope": "foundry",
+ "src": "Kustvakt"
+ }
+ ]
+ }
+ }`,
+ expected: &ast.Token{
+ Wrap: &ast.Term{
+ Foundry: "opennlp",
+ Key: "Baum",
+ Layer: "orth",
+ Match: ast.MatchEqual,
+ Rewrites: []ast.Rewrite{
+ {
+ Comment: "Default foundry has been added.",
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ },
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "Parse term group with rewrites",
+ input: `{
+ "@type": "koral:termGroup",
+ "operands": [
+ {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "DET",
+ "layer": "p",
+ "match": "match:eq"
+ },
+ {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "AdjType",
+ "layer": "m",
+ "match": "match:eq",
+ "value": "Pdt"
+ }
+ ],
+ "relation": "relation:and",
+ "rewrites": [
+ {
+ "@type": "koral:rewrite",
+ "_comment": "Default foundry has been added.",
+ "editor": "Kustvakt",
+ "operation": "operation:injection",
+ "scope": "foundry",
+ "src": "Kustvakt"
+ }
+ ]
+ }`,
+ expected: &ast.TermGroup{
+ Operands: []ast.Node{
+ &ast.Term{
+ Foundry: "opennlp",
+ Key: "DET",
+ Layer: "p",
+ Match: ast.MatchEqual,
+ },
+ &ast.Term{
+ Foundry: "opennlp",
+ Key: "AdjType",
+ Layer: "m",
+ Match: ast.MatchEqual,
+ Value: "Pdt",
+ },
+ },
+ Relation: ast.AndRelation,
+ Rewrites: []ast.Rewrite{
+ {
+ Comment: "Default foundry has been added.",
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "Parse term with rewrites",
+ input: `{
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "DET",
+ "layer": "p",
+ "match": "match:eq",
+ "rewrites": [
+ {
+ "@type": "koral:rewrite",
+ "_comment": "Default foundry has been added.",
+ "editor": "Kustvakt",
+ "operation": "operation:injection",
+ "scope": "foundry",
+ "src": "Kustvakt"
+ }
+ ]
+ }`,
+ expected: &ast.Term{
+ Foundry: "opennlp",
+ Key: "DET",
+ Layer: "p",
+ Match: ast.MatchEqual,
+ Rewrites: []ast.Rewrite{
+ {
+ Comment: "Default foundry has been added.",
+ Editor: "Kustvakt",
+ Operation: "operation:injection",
+ Scope: "foundry",
+ Src: "Kustvakt",
+ },
+ },
+ },
+ wantErr: false,
+ },
+ {
name: "Invalid JSON",
input: `{"invalid": json`,
wantErr: true,
@@ -330,6 +469,105 @@
}
}
+func TestParseJSONErrors(t *testing.T) {
+ tests := []struct {
+ name string
+ input string
+ wantErr bool
+ }{
+ {
+ name: "Empty JSON",
+ input: "{}",
+ wantErr: true,
+ },
+ {
+ name: "Invalid JSON",
+ input: "{",
+ wantErr: true,
+ },
+ {
+ name: "Token without wrap",
+ input: `{
+ "@type": "koral:token"
+ }`,
+ wantErr: true,
+ },
+ {
+ name: "Term without key",
+ input: `{
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "layer": "p",
+ "match": "match:eq"
+ }`,
+ wantErr: true,
+ },
+ {
+ name: "TermGroup without operands",
+ input: `{
+ "@type": "koral:termGroup",
+ "relation": "relation:and"
+ }`,
+ wantErr: true,
+ },
+ {
+ name: "TermGroup without relation",
+ input: `{
+ "@type": "koral:termGroup",
+ "operands": [
+ {
+ "@type": "koral:term",
+ "key": "DET",
+ "foundry": "opennlp",
+ "layer": "p",
+ "match": "match:eq"
+ }
+ ]
+ }`,
+ wantErr: true,
+ },
+ {
+ name: "Invalid match type",
+ input: `{
+ "@type": "koral:term",
+ "key": "DET",
+ "foundry": "opennlp",
+ "layer": "p",
+ "match": "match:invalid"
+ }`,
+ wantErr: true,
+ },
+ {
+ name: "Invalid relation type",
+ input: `{
+ "@type": "koral:termGroup",
+ "operands": [
+ {
+ "@type": "koral:term",
+ "key": "DET",
+ "foundry": "opennlp",
+ "layer": "p",
+ "match": "match:eq"
+ }
+ ],
+ "relation": "relation:invalid"
+ }`,
+ wantErr: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ _, err := ParseJSON([]byte(tt.input))
+ if tt.wantErr {
+ assert.Error(t, err)
+ } else {
+ assert.NoError(t, err)
+ }
+ })
+ }
+}
+
func TestSerializeToJSON(t *testing.T) {
tests := []struct {
name string
@@ -721,29 +959,6 @@
wantErr: false,
},
{
- name: "Invalid match type",
- input: `{
- "@type": "koral:term",
- "key": "DET",
- "match": "match:invalid"
- }`,
- wantErr: true,
- },
- {
- name: "Invalid relation type",
- input: `{
- "@type": "koral:termGroup",
- "operands": [
- {
- "@type": "koral:term",
- "key": "DET"
- }
- ],
- "relation": "relation:invalid"
- }`,
- wantErr: true,
- },
- {
name: "Empty operands in term group",
input: `{
"@type": "koral:termGroup",