Support corpus mappings
Change-Id: I25e987b0ca668a1cf733424b22edb4f0fca37bf2
diff --git a/README.md b/README.md
index 24dc3af..b4daefd 100644
--- a/README.md
+++ b/README.md
@@ -94,9 +94,22 @@
These values are applied during configuration parsing. When using only individual mapping files (`-m` flags), default values are used unless overridden by command line arguments.
-### Mapping Rules
-Each mapping rule consists of two patterns separated by `<>`. The patterns can be:
+### Corpus mapping rules (type: corpus)
+
+Corpus mapping rules use `key=value <> key=value` syntax for rewriting `koral:doc` / `koral:docGroup` structures in the `corpus`/`collection` section of a KoralQuery request, and enriching `fields` arrays in responses.
+
+- Simple fields: `textClass=novel <> genre=fiction`
+- Match types: `pubDate=2020:geq <> yearFrom=2020:geq` (eq, ne, geq, leq, contains, excludes)
+- Value types: `pubDate=2020-01#date <> year=2020#string` (string, regex, date)
+- Regex matching: `textClass=wissenschaft.*#regex <> genre=science`
+- Groups (AND/OR): `(textClass=novel & pubDate=2020) <> genre=fiction`
+
+(Supported `@type` aliases: `koral:field` for `koral:doc`, `koral:fieldGroup` for `koral:docGroup`).
+
+### Annotation mapping rules (type: annotation)
+
+Each annotation mapping rule consists of two patterns separated by `<>`. The patterns can be:
- Simple terms (e.g. `[key]`, `[layer=key]`, `[foundry/*=key]`, `[foundry/layer=key]` or `[foundry/layer=key:value]`)
- Complex terms with AND/OR relations: `[term1 & term2]` or `[term1 | term2]` or `[term1 | (term2 & term3)]`
@@ -244,15 +257,16 @@
- [x] Integration of multiple mapping files
- [x] Response rewriting
- [ ] Support for negation
-- [ ] Support corpus mappings
+- [x] Support corpus mappings
- [ ] Support chaining of mappings
## COPYRIGHT AND LICENSE
-*Disclaimer*: This software was developed as an experiment with major assistance by AI (mainly Claude 3.5-sonnet and Claude 4-sonnet).
+*Disclaimer*: This software was developed as an experiment with major assistance by AI
+(mainly Claude 3.5-sonnet and Claude 4-sonnet, starting with 0.1.1: Claude 4.6 Opus and GPT 5.3 Codex).
The code should not be used as an example on how to create services as Kalamar plugins.
-Copyright (C) 2025, [IDS Mannheim](https://www.ids-mannheim.de/)<br>
+Copyright (C) 2025-2026, [IDS Mannheim](https://www.ids-mannheim.de/)<br>
Author: [Nils Diewald](https://www.nils-diewald.de/)
Koral-Mapper is free software published under the
diff --git a/config/config.go b/config/config.go
index d245461..cdc1e0f 100644
--- a/config/config.go
+++ b/config/config.go
@@ -24,6 +24,7 @@
// MappingList represents a list of mapping rules with metadata
type MappingList struct {
ID string `yaml:"id"`
+ Type string `yaml:"type,omitempty"` // "annotation" (default) or "corpus"
Description string `yaml:"desc,omitempty"`
FoundryA string `yaml:"foundryA,omitempty"`
LayerA string `yaml:"layerA,omitempty"`
@@ -32,6 +33,28 @@
Mappings []MappingRule `yaml:"mappings"`
}
+// IsCorpus returns true if the mapping list type is "corpus".
+func (list *MappingList) IsCorpus() bool {
+ return list.Type == "corpus"
+}
+
+// ParseCorpusMappings parses all mapping rules as corpus rules.
+func (list *MappingList) ParseCorpusMappings() ([]*parser.CorpusMappingResult, error) {
+ corpusParser := parser.NewCorpusParser()
+ results := make([]*parser.CorpusMappingResult, len(list.Mappings))
+ for i, rule := range list.Mappings {
+ if rule == "" {
+ return nil, fmt.Errorf("empty corpus mapping rule at index %d in list '%s'", i, list.ID)
+ }
+ result, err := corpusParser.ParseMapping(string(rule))
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse corpus mapping rule %d in list '%s': %w", i, list.ID, err)
+ }
+ results[i] = result
+ }
+ return results, nil
+}
+
// MappingConfig represents the root configuration containing multiple mapping lists
type MappingConfig struct {
SDK string `yaml:"sdk,omitempty"`
diff --git a/config/config_test.go b/config/config_test.go
index cc14416..845ebdd 100644
--- a/config/config_test.go
+++ b/config/config_test.go
@@ -868,3 +868,81 @@
assert.Equal(t, defaultServer, config.Server)
assert.Equal(t, defaultServiceURL, config.ServiceURL)
}
+
+func TestCorpusMappingListType(t *testing.T) {
+ content := `
+lists:
+- id: corpus-class-mapping
+ type: corpus
+ desc: Maps textClass values to genre field
+ mappings:
+ - "textClass=novel <> genre=fiction"
+ - "textClass=science <> genre=nonfiction"
+- id: annotation-mapper
+ mappings:
+ - "[A] <> [B]"
+`
+ tmpfile, err := os.CreateTemp("", "config-corpus-*.yaml")
+ require.NoError(t, err)
+ defer os.Remove(tmpfile.Name())
+
+ _, err = tmpfile.WriteString(content)
+ require.NoError(t, err)
+ err = tmpfile.Close()
+ require.NoError(t, err)
+
+ config, err := LoadFromSources(tmpfile.Name(), nil)
+ require.NoError(t, err)
+ require.Len(t, config.Lists, 2)
+
+ assert.Equal(t, "corpus", config.Lists[0].Type)
+ assert.True(t, config.Lists[0].IsCorpus())
+
+ assert.Equal(t, "", config.Lists[1].Type)
+ assert.False(t, config.Lists[1].IsCorpus())
+}
+
+func TestParseCorpusMappings(t *testing.T) {
+ list := &MappingList{
+ ID: "test-corpus",
+ Type: "corpus",
+ Mappings: []MappingRule{
+ "textClass=novel <> genre=fiction",
+ "(textClass=novel & pubDate=2020:geq#date) <> genre=recentfiction",
+ },
+ }
+
+ results, err := list.ParseCorpusMappings()
+ require.NoError(t, err)
+ require.Len(t, results, 2)
+
+ // Verify simple field rule
+ require.NotNil(t, results[0].Upper)
+ require.NotNil(t, results[0].Lower)
+
+ // Verify group rule
+ require.NotNil(t, results[1].Upper)
+ require.NotNil(t, results[1].Lower)
+}
+
+func TestParseCorpusMappingsErrors(t *testing.T) {
+ list := &MappingList{
+ ID: "test-corpus",
+ Type: "corpus",
+ Mappings: []MappingRule{""},
+ }
+
+ _, err := list.ParseCorpusMappings()
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "empty corpus mapping rule")
+
+ list2 := &MappingList{
+ ID: "test-corpus",
+ Type: "corpus",
+ Mappings: []MappingRule{"invalid rule without separator"},
+ }
+
+ _, err = list2.ParseCorpusMappings()
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "failed to parse corpus mapping rule")
+}
diff --git a/mapper/corpus.go b/mapper/corpus.go
new file mode 100644
index 0000000..d45dc92
--- /dev/null
+++ b/mapper/corpus.go
@@ -0,0 +1,344 @@
+package mapper
+
+import (
+ "regexp"
+
+ "github.com/KorAP/Koral-Mapper/parser"
+)
+
+// applyCorpusQueryMappings processes corpus/collection section with corpus rules.
+func (m *Mapper) applyCorpusQueryMappings(mappingID string, opts MappingOptions, jsonData any) (any, error) {
+ rules := m.parsedCorpusRules[mappingID]
+
+ jsonMap, ok := jsonData.(map[string]any)
+ if !ok {
+ return jsonData, nil
+ }
+
+ // Find corpus or collection attribute
+ corpusKey := ""
+ if _, exists := jsonMap["corpus"]; exists {
+ corpusKey = "corpus"
+ } else if _, exists := jsonMap["collection"]; exists {
+ corpusKey = "collection"
+ }
+
+ if corpusKey == "" {
+ return jsonData, nil
+ }
+
+ corpusData, ok := jsonMap[corpusKey].(map[string]any)
+ if !ok {
+ return jsonData, nil
+ }
+
+ result := shallowCopyMap(jsonMap)
+ rewritten := m.rewriteCorpusNode(corpusData, rules, opts)
+ result[corpusKey] = rewritten
+
+ return result, nil
+}
+
+// rewriteCorpusNode recursively walks a corpus tree and applies matching rules.
+func (m *Mapper) rewriteCorpusNode(node map[string]any, rules []*parser.CorpusMappingResult, opts MappingOptions) any {
+ atType, _ := node["@type"].(string)
+
+ switch atType {
+ case "koral:doc", "koral:field":
+ return m.rewriteCorpusDoc(node, rules, opts)
+ case "koral:docGroup", "koral:fieldGroup":
+ return m.rewriteCorpusDocGroup(node, rules, opts)
+ case "koral:docGroupRef":
+ return node
+ default:
+ return node
+ }
+}
+
+// rewriteCorpusDoc attempts to match a koral:doc node against rules and replace it.
+func (m *Mapper) rewriteCorpusDoc(node map[string]any, rules []*parser.CorpusMappingResult, opts MappingOptions) any {
+ for _, rule := range rules {
+ var pattern, replacement parser.CorpusNode
+ if opts.Direction == AtoB {
+ pattern, replacement = rule.Upper, rule.Lower
+ } else {
+ pattern, replacement = rule.Lower, rule.Upper
+ }
+
+ patternField, ok := pattern.(*parser.CorpusField)
+ if !ok {
+ continue
+ }
+
+ if !matchCorpusField(patternField, node) {
+ continue
+ }
+
+ replaced := buildReplacementFromNode(replacement, node)
+
+ if opts.AddRewrites {
+ addCorpusRewrite(replaced, node)
+ }
+
+ return replaced
+ }
+
+ return node
+}
+
+// rewriteCorpusDocGroup recursively rewrites operands of a koral:docGroup.
+func (m *Mapper) rewriteCorpusDocGroup(node map[string]any, rules []*parser.CorpusMappingResult, opts MappingOptions) any {
+ result := shallowCopyMap(node)
+
+ operandsRaw, ok := node["operands"].([]any)
+ if !ok {
+ return result
+ }
+
+ newOperands := make([]any, len(operandsRaw))
+ for i, opRaw := range operandsRaw {
+ opMap, ok := opRaw.(map[string]any)
+ if !ok {
+ newOperands[i] = opRaw
+ continue
+ }
+ newOperands[i] = m.rewriteCorpusNode(opMap, rules, opts)
+ }
+ result["operands"] = newOperands
+
+ return result
+}
+
+// matchCorpusField checks if a koral:doc JSON node matches a CorpusField pattern.
+func matchCorpusField(pattern *parser.CorpusField, doc map[string]any) bool {
+ docKey, _ := doc["key"].(string)
+ if docKey != pattern.Key {
+ return false
+ }
+
+ docValue, _ := doc["value"].(string)
+ if pattern.Type == "regex" {
+ re, err := regexp.Compile("^" + pattern.Value + "$")
+ if err != nil {
+ return false
+ }
+ if !re.MatchString(docValue) {
+ return false
+ }
+ } else if docValue != pattern.Value {
+ return false
+ }
+
+ if pattern.Match != "" {
+ docMatch, _ := doc["match"].(string)
+ expected := "match:" + pattern.Match
+ if docMatch != expected {
+ return false
+ }
+ }
+
+ if pattern.Type != "" && pattern.Type != "regex" {
+ docType, _ := doc["type"].(string)
+ expected := "type:" + pattern.Type
+ if docType != "" && docType != expected {
+ return false
+ }
+ }
+
+ return true
+}
+
+// buildReplacementFromNode builds a replacement JSON structure from a CorpusNode pattern.
+// Preserves match and type from the original doc when the rule doesn't specify them.
+func buildReplacementFromNode(replacement parser.CorpusNode, originalDoc map[string]any) any {
+ switch r := replacement.(type) {
+ case *parser.CorpusField:
+ result := map[string]any{
+ "@type": originalDoc["@type"],
+ "key": r.Key,
+ "value": r.Value,
+ }
+
+ if r.Match != "" {
+ result["match"] = "match:" + r.Match
+ } else if m, ok := originalDoc["match"]; ok {
+ result["match"] = m
+ }
+
+ if r.Type != "" {
+ result["type"] = "type:" + r.Type
+ } else if t, ok := originalDoc["type"]; ok {
+ result["type"] = t
+ }
+
+ return result
+
+ case *parser.CorpusGroup:
+ operands := make([]any, len(r.Operands))
+ for i, op := range r.Operands {
+ operands[i] = buildReplacementFromNode(op, originalDoc)
+ }
+ return map[string]any{
+ "@type": "koral:docGroup",
+ "operation": "operation:" + r.Operation,
+ "operands": operands,
+ }
+
+ default:
+ return originalDoc
+ }
+}
+
+// addCorpusRewrite adds a koral:rewrite annotation to the replaced node.
+func addCorpusRewrite(replaced any, original map[string]any) {
+ replacedMap, ok := replaced.(map[string]any)
+ if !ok {
+ return
+ }
+
+ origKey, _ := original["key"].(string)
+ newKey, _ := replacedMap["key"].(string)
+
+ var rewrite map[string]any
+ if origKey != newKey && origKey != "" {
+ rewrite = newRewriteEntry("key", origKey)
+ } else {
+ origValue, _ := original["value"].(string)
+ rewrite = newRewriteEntry("value", origValue)
+ }
+
+ replacedMap["rewrites"] = []any{rewrite}
+}
+
+// applyCorpusResponseMappings processes fields arrays with corpus rules.
+func (m *Mapper) applyCorpusResponseMappings(mappingID string, opts MappingOptions, jsonData any) (any, error) {
+ rules := m.parsedCorpusRules[mappingID]
+
+ jsonMap, ok := jsonData.(map[string]any)
+ if !ok {
+ return jsonData, nil
+ }
+
+ fieldsRaw, exists := jsonMap["fields"]
+ if !exists {
+ return jsonData, nil
+ }
+
+ fields, ok := fieldsRaw.([]any)
+ if !ok {
+ return jsonData, nil
+ }
+
+ var newFields []any
+ for _, fieldRaw := range fields {
+ newFields = append(newFields, fieldRaw)
+
+ fieldMap, ok := fieldRaw.(map[string]any)
+ if !ok {
+ continue
+ }
+
+ atType, _ := fieldMap["@type"].(string)
+ if atType != "koral:field" && atType != "koral:doc" {
+ continue
+ }
+
+ fieldKey, _ := fieldMap["key"].(string)
+ fieldValue := fieldMap["value"]
+
+ mapped := m.matchFieldAndCollect(fieldKey, fieldValue, rules, opts)
+ newFields = append(newFields, mapped...)
+ }
+
+ result := shallowCopyMap(jsonMap)
+ result["fields"] = newFields
+ return result, nil
+}
+
+// matchFieldAndCollect matches a field's key/value against rules and returns mapped entries.
+// For array values, each element is matched individually.
+func (m *Mapper) matchFieldAndCollect(key string, value any, rules []*parser.CorpusMappingResult, opts MappingOptions) []any {
+ var results []any
+
+ switch v := value.(type) {
+ case string:
+ results = append(results, m.matchSingleValue(key, v, rules, opts)...)
+ case []any:
+ for _, elem := range v {
+ if s, ok := elem.(string); ok {
+ results = append(results, m.matchSingleValue(key, s, rules, opts)...)
+ }
+ }
+ }
+
+ return results
+}
+
+// matchSingleValue checks a single key+value pair against all rules and returns mapped field entries.
+func (m *Mapper) matchSingleValue(key, value string, rules []*parser.CorpusMappingResult, opts MappingOptions) []any {
+ var results []any
+
+ pseudoDoc := map[string]any{
+ "key": key,
+ "value": value,
+ }
+
+ for _, rule := range rules {
+ var pattern, replacement parser.CorpusNode
+ if opts.Direction == AtoB {
+ pattern, replacement = rule.Upper, rule.Lower
+ } else {
+ pattern, replacement = rule.Lower, rule.Upper
+ }
+
+ patternField, ok := pattern.(*parser.CorpusField)
+ if !ok {
+ continue
+ }
+
+ if !matchCorpusField(patternField, pseudoDoc) {
+ continue
+ }
+
+ results = append(results, collectReplacementFields(replacement)...)
+ }
+
+ return results
+}
+
+// collectReplacementFields flattens a replacement CorpusNode into individual mapped field entries.
+func collectReplacementFields(node parser.CorpusNode) []any {
+ var results []any
+
+ switch n := node.(type) {
+ case *parser.CorpusField:
+ entry := map[string]any{
+ "@type": "koral:field",
+ "key": n.Key,
+ "value": n.Value,
+ "mapped": true,
+ }
+ if n.Type != "" {
+ entry["type"] = "type:" + n.Type
+ } else {
+ entry["type"] = "type:string"
+ }
+ results = append(results, entry)
+
+ case *parser.CorpusGroup:
+ for _, op := range n.Operands {
+ results = append(results, collectReplacementFields(op)...)
+ }
+ }
+
+ return results
+}
+
+func shallowCopyMap(m map[string]any) map[string]any {
+ result := make(map[string]any, len(m))
+ for k, v := range m {
+ result[k] = v
+ }
+ return result
+}
+
diff --git a/mapper/corpus_test.go b/mapper/corpus_test.go
new file mode 100644
index 0000000..4f607e7
--- /dev/null
+++ b/mapper/corpus_test.go
@@ -0,0 +1,686 @@
+package mapper
+
+import (
+ "testing"
+
+ "github.com/KorAP/Koral-Mapper/config"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func newCorpusMapper(t *testing.T, rules ...string) *Mapper {
+ t.Helper()
+ mappingRules := make([]config.MappingRule, len(rules))
+ for i, r := range rules {
+ mappingRules[i] = config.MappingRule(r)
+ }
+ m, err := NewMapper([]config.MappingList{{
+ ID: "corpus-test",
+ Type: "corpus",
+ Mappings: mappingRules,
+ }})
+ require.NoError(t, err)
+ return m
+}
+
+// --- Corpus query mapping tests ---
+
+func TestCorpusQuerySimpleFieldRewrite(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "novel",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "koral:doc", corpus["@type"])
+ assert.Equal(t, "genre", corpus["key"])
+ assert.Equal(t, "fiction", corpus["value"])
+ assert.Equal(t, "match:eq", corpus["match"])
+}
+
+func TestCorpusQueryNoMatch(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "science",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "textClass", corpus["key"])
+ assert.Equal(t, "science", corpus["value"])
+}
+
+func TestCorpusQueryBtoA(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "genre",
+ "value": "fiction",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: BtoA}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "textClass", corpus["key"])
+ assert.Equal(t, "novel", corpus["value"])
+}
+
+func TestCorpusQueryDocGroupRecursive(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:docGroup",
+ "operation": "operation:and",
+ "operands": []any{
+ map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "novel",
+ "match": "match:eq",
+ },
+ map[string]any{
+ "@type": "koral:doc",
+ "key": "author",
+ "value": "Fontane",
+ "match": "match:eq",
+ },
+ },
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "koral:docGroup", corpus["@type"])
+ assert.Equal(t, "operation:and", corpus["operation"])
+
+ operands := corpus["operands"].([]any)
+ require.Len(t, operands, 2)
+
+ first := operands[0].(map[string]any)
+ assert.Equal(t, "genre", first["key"])
+ assert.Equal(t, "fiction", first["value"])
+
+ second := operands[1].(map[string]any)
+ assert.Equal(t, "author", second["key"])
+ assert.Equal(t, "Fontane", second["value"])
+}
+
+func TestCorpusQueryDocGroupRefPassthrough(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:docGroupRef",
+ "ref": "https://korap.ids-mannheim.de/@ndiewald/MyCorpus",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "koral:docGroupRef", corpus["@type"])
+ assert.Equal(t, "https://korap.ids-mannheim.de/@ndiewald/MyCorpus", corpus["ref"])
+}
+
+func TestCorpusQueryFieldAlias(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:field",
+ "key": "textClass",
+ "value": "novel",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "genre", corpus["key"])
+ assert.Equal(t, "fiction", corpus["value"])
+}
+
+func TestCorpusQueryFieldGroupAlias(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:fieldGroup",
+ "operation": "operation:and",
+ "operands": []any{
+ map[string]any{
+ "@type": "koral:field",
+ "key": "textClass",
+ "value": "novel",
+ },
+ },
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ operands := corpus["operands"].([]any)
+ first := operands[0].(map[string]any)
+ assert.Equal(t, "genre", first["key"])
+}
+
+func TestCorpusQueryCollectionAttribute(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "collection": map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "novel",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["collection"].(map[string]any)
+ assert.Equal(t, "genre", corpus["key"])
+ assert.Equal(t, "fiction", corpus["value"])
+}
+
+func TestCorpusQuerySingleToGroupReplacement(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> (genre=fiction & type=book)")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "novel",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "koral:docGroup", corpus["@type"])
+ assert.Equal(t, "operation:and", corpus["operation"])
+
+ operands := corpus["operands"].([]any)
+ require.Len(t, operands, 2)
+ assert.Equal(t, "genre", operands[0].(map[string]any)["key"])
+ assert.Equal(t, "type", operands[1].(map[string]any)["key"])
+}
+
+func TestCorpusQueryRegexMatch(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=wissenschaft.*#regex <> genre=science")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "wissenschaft-populaer",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "genre", corpus["key"])
+ assert.Equal(t, "science", corpus["value"])
+}
+
+func TestCorpusQueryRegexNoMatch(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=wissenschaft.*#regex <> genre=science")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "belletristik",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "textClass", corpus["key"])
+ assert.Equal(t, "belletristik", corpus["value"])
+}
+
+func TestCorpusQueryMatchTypeFilter(t *testing.T) {
+ m := newCorpusMapper(t, "pubDate=2020:geq <> yearFrom=2020:geq")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "pubDate",
+ "value": "2020",
+ "match": "match:geq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "yearFrom", corpus["key"])
+ assert.Equal(t, "match:geq", corpus["match"])
+}
+
+func TestCorpusQueryMatchTypeFilterNoMatchTest(t *testing.T) {
+ m := newCorpusMapper(t, "pubDate=2020 <> yearFrom=2020")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "pubDate",
+ "value": "2020",
+ "match": "match:geq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "yearFrom", corpus["key"])
+ assert.Equal(t, "match:geq", corpus["match"])
+}
+
+func TestCorpusQueryMatchTypeFilterNoMatch(t *testing.T) {
+ m := newCorpusMapper(t, "pubDate=2020:geq <> yearFrom=2020:geq")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "pubDate",
+ "value": "2020",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "pubDate", corpus["key"])
+}
+
+func TestCorpusQueryRewriteAnnotation(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "novel",
+ "match": "match:eq",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB, AddRewrites: true}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "genre", corpus["key"])
+
+ rewrites, ok := corpus["rewrites"].([]any)
+ require.True(t, ok)
+ require.Len(t, rewrites, 1)
+
+ rewrite := rewrites[0].(map[string]any)
+ assert.Equal(t, "koral:rewrite", rewrite["@type"])
+ assert.Equal(t, "Koral-Mapper", rewrite["editor"])
+}
+
+func TestCorpusQueryPreservesMatchTypeFromOriginal(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "novel",
+ "match": "match:contains",
+ "type": "type:string",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "genre", corpus["key"])
+ assert.Equal(t, "match:contains", corpus["match"])
+ assert.Equal(t, "type:string", corpus["type"])
+}
+
+func TestCorpusQueryNoCorpusSection(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "query": map[string]any{"@type": "koral:token"},
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+ assert.Equal(t, input, result)
+}
+
+func TestCorpusQueryMultipleRules(t *testing.T) {
+ m := newCorpusMapper(t,
+ "textClass=novel <> genre=fiction",
+ "textClass=science <> genre=nonfiction",
+ )
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:docGroup",
+ "operation": "operation:or",
+ "operands": []any{
+ map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "novel",
+ },
+ map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "science",
+ },
+ },
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ operands := corpus["operands"].([]any)
+ require.Len(t, operands, 2)
+ assert.Equal(t, "genre", operands[0].(map[string]any)["key"])
+ assert.Equal(t, "fiction", operands[0].(map[string]any)["value"])
+ assert.Equal(t, "genre", operands[1].(map[string]any)["key"])
+ assert.Equal(t, "nonfiction", operands[1].(map[string]any)["value"])
+}
+
+func TestCorpusQueryNestedDocGroups(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:docGroup",
+ "operation": "operation:and",
+ "operands": []any{
+ map[string]any{
+ "@type": "koral:docGroup",
+ "operation": "operation:or",
+ "operands": []any{
+ map[string]any{
+ "@type": "koral:doc",
+ "key": "textClass",
+ "value": "novel",
+ },
+ },
+ },
+ map[string]any{
+ "@type": "koral:doc",
+ "key": "author",
+ "value": "Fontane",
+ },
+ },
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ outerOperands := corpus["operands"].([]any)
+ innerGroup := outerOperands[0].(map[string]any)
+ innerOperands := innerGroup["operands"].([]any)
+ assert.Equal(t, "genre", innerOperands[0].(map[string]any)["key"])
+}
+
+// --- Corpus response mapping tests ---
+
+func TestCorpusResponseSimpleFieldEnrichment(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "fields": []any{
+ map[string]any{
+ "@type": "koral:field",
+ "key": "genre",
+ "value": "fiction",
+ "type": "type:string",
+ },
+ },
+ }
+ result, err := m.ApplyResponseMappings("corpus-test", MappingOptions{Direction: BtoA}, input)
+ require.NoError(t, err)
+
+ fields := result.(map[string]any)["fields"].([]any)
+ require.Len(t, fields, 2)
+
+ original := fields[0].(map[string]any)
+ assert.Equal(t, "genre", original["key"])
+
+ mapped := fields[1].(map[string]any)
+ assert.Equal(t, "textClass", mapped["key"])
+ assert.Equal(t, "novel", mapped["value"])
+ assert.Equal(t, true, mapped["mapped"])
+}
+
+func TestCorpusResponseNoMatch(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "fields": []any{
+ map[string]any{
+ "@type": "koral:field",
+ "key": "author",
+ "value": "Fontane",
+ "type": "type:string",
+ },
+ },
+ }
+ result, err := m.ApplyResponseMappings("corpus-test", MappingOptions{Direction: BtoA}, input)
+ require.NoError(t, err)
+
+ fields := result.(map[string]any)["fields"].([]any)
+ require.Len(t, fields, 1)
+}
+
+func TestCorpusResponseMultiValuedField(t *testing.T) {
+ m := newCorpusMapper(t,
+ "textClass=wissenschaft <> genre=science",
+ "textClass=populaerwissenschaft <> genre=popsci",
+ )
+
+ input := map[string]any{
+ "fields": []any{
+ map[string]any{
+ "@type": "koral:field",
+ "key": "textClass",
+ "value": []any{"wissenschaft", "populaerwissenschaft"},
+ "type": "type:keywords",
+ },
+ },
+ }
+ result, err := m.ApplyResponseMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ fields := result.(map[string]any)["fields"].([]any)
+ require.Len(t, fields, 3)
+
+ mapped1 := fields[1].(map[string]any)
+ assert.Equal(t, "genre", mapped1["key"])
+ assert.Equal(t, "science", mapped1["value"])
+ assert.Equal(t, true, mapped1["mapped"])
+
+ mapped2 := fields[2].(map[string]any)
+ assert.Equal(t, "genre", mapped2["key"])
+ assert.Equal(t, "popsci", mapped2["value"])
+ assert.Equal(t, true, mapped2["mapped"])
+}
+
+func TestCorpusResponseRegexMatch(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=wissenschaft.*#regex <> genre=science")
+
+ input := map[string]any{
+ "fields": []any{
+ map[string]any{
+ "@type": "koral:field",
+ "key": "textClass",
+ "value": "wissenschaft-populaer",
+ "type": "type:string",
+ },
+ },
+ }
+ result, err := m.ApplyResponseMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ fields := result.(map[string]any)["fields"].([]any)
+ require.Len(t, fields, 2)
+
+ mapped := fields[1].(map[string]any)
+ assert.Equal(t, "genre", mapped["key"])
+ assert.Equal(t, "science", mapped["value"])
+}
+
+func TestCorpusResponseDocTypeAlias(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "fields": []any{
+ map[string]any{
+ "@type": "koral:doc",
+ "key": "genre",
+ "value": "fiction",
+ "type": "type:string",
+ },
+ },
+ }
+ result, err := m.ApplyResponseMappings("corpus-test", MappingOptions{Direction: BtoA}, input)
+ require.NoError(t, err)
+
+ fields := result.(map[string]any)["fields"].([]any)
+ require.Len(t, fields, 2)
+
+ mapped := fields[1].(map[string]any)
+ assert.Equal(t, "textClass", mapped["key"])
+}
+
+func TestCorpusResponseGroupReplacement(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> (genre=fiction & type=book)")
+
+ input := map[string]any{
+ "fields": []any{
+ map[string]any{
+ "@type": "koral:field",
+ "key": "textClass",
+ "value": "novel",
+ "type": "type:string",
+ },
+ },
+ }
+ result, err := m.ApplyResponseMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ fields := result.(map[string]any)["fields"].([]any)
+ require.Len(t, fields, 3)
+
+ mapped1 := fields[1].(map[string]any)
+ assert.Equal(t, "genre", mapped1["key"])
+ assert.Equal(t, "fiction", mapped1["value"])
+ assert.Equal(t, true, mapped1["mapped"])
+
+ mapped2 := fields[2].(map[string]any)
+ assert.Equal(t, "type", mapped2["key"])
+ assert.Equal(t, "book", mapped2["value"])
+ assert.Equal(t, true, mapped2["mapped"])
+}
+
+func TestCorpusResponseNoFieldsSection(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "snippet": "<span>test</span>",
+ }
+ result, err := m.ApplyResponseMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+ assert.Equal(t, input, result)
+}
+
+func TestCorpusResponseDirectionAtoB(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+
+ input := map[string]any{
+ "fields": []any{
+ map[string]any{
+ "@type": "koral:field",
+ "key": "textClass",
+ "value": "novel",
+ "type": "type:string",
+ },
+ },
+ }
+ result, err := m.ApplyResponseMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ fields := result.(map[string]any)["fields"].([]any)
+ require.Len(t, fields, 2)
+
+ mapped := fields[1].(map[string]any)
+ assert.Equal(t, "genre", mapped["key"])
+ assert.Equal(t, "fiction", mapped["value"])
+}
+
+func TestCorpusQueryValueTypeInReplacement(t *testing.T) {
+ m := newCorpusMapper(t, "pubDate=2020-01#date <> publicationYear=2020#string")
+
+ input := map[string]any{
+ "corpus": map[string]any{
+ "@type": "koral:doc",
+ "key": "pubDate",
+ "value": "2020-01",
+ "match": "match:eq",
+ "type": "type:date",
+ },
+ }
+ result, err := m.ApplyQueryMappings("corpus-test", MappingOptions{Direction: AtoB}, input)
+ require.NoError(t, err)
+
+ corpus := result.(map[string]any)["corpus"].(map[string]any)
+ assert.Equal(t, "publicationYear", corpus["key"])
+ assert.Equal(t, "2020", corpus["value"])
+ assert.Equal(t, "type:string", corpus["type"])
+}
+
+func TestCorpusQueryMappingListNotFound(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+ _, err := m.ApplyQueryMappings("nonexistent", MappingOptions{Direction: AtoB}, map[string]any{})
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "not found")
+}
+
+func TestCorpusResponseMappingListNotFound(t *testing.T) {
+ m := newCorpusMapper(t, "textClass=novel <> genre=fiction")
+ _, err := m.ApplyResponseMappings("nonexistent", MappingOptions{Direction: AtoB}, map[string]any{})
+ assert.Error(t, err)
+ assert.Contains(t, err.Error(), "not found")
+}
diff --git a/mapper/mapper.go b/mapper/mapper.go
index d11ae01..e55e77a 100644
--- a/mapper/mapper.go
+++ b/mapper/mapper.go
@@ -13,8 +13,25 @@
const (
AtoB Direction = true
BtoA Direction = false
+
+ RewriteEditor = "Koral-Mapper"
)
+// newRewriteEntry creates a koral:rewrite annotation entry.
+func newRewriteEntry(scope string, original any) map[string]any {
+ r := map[string]any{
+ "@type": "koral:rewrite",
+ "editor": RewriteEditor,
+ }
+ if scope != "" {
+ r["scope"] = scope
+ }
+ if original != nil {
+ r["original"] = original
+ }
+ return r
+}
+
// String converts the Direction to its string representation
func (d Direction) String() string {
if d {
@@ -37,15 +54,17 @@
// Mapper handles the application of mapping rules to JSON objects
type Mapper struct {
- mappingLists map[string]*config.MappingList
- parsedRules map[string][]*parser.MappingResult
+ mappingLists map[string]*config.MappingList
+ parsedQueryRules map[string][]*parser.MappingResult
+ parsedCorpusRules map[string][]*parser.CorpusMappingResult
}
// NewMapper creates a new Mapper instance from a list of MappingLists
func NewMapper(lists []config.MappingList) (*Mapper, error) {
m := &Mapper{
- mappingLists: make(map[string]*config.MappingList),
- parsedRules: make(map[string][]*parser.MappingResult),
+ mappingLists: make(map[string]*config.MappingList),
+ parsedQueryRules: make(map[string][]*parser.MappingResult),
+ parsedCorpusRules: make(map[string][]*parser.CorpusMappingResult),
}
// Store mapping lists by ID
@@ -54,16 +73,22 @@
return nil, fmt.Errorf("duplicate mapping list ID found: %s", list.ID)
}
- // Create a copy of the list to store
listCopy := list
m.mappingLists[list.ID] = &listCopy
- // Parse the rules immediately
- parsedRules, err := list.ParseMappings()
- if err != nil {
- return nil, fmt.Errorf("failed to parse mappings for list %s: %w", list.ID, err)
+ if list.IsCorpus() {
+ corpusRules, err := list.ParseCorpusMappings()
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse corpus mappings for list %s: %w", list.ID, err)
+ }
+ m.parsedCorpusRules[list.ID] = corpusRules
+ } else {
+ queryRules, err := list.ParseMappings()
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse mappings for list %s: %w", list.ID, err)
+ }
+ m.parsedQueryRules[list.ID] = queryRules
}
- m.parsedRules[list.ID] = parsedRules
}
return m, nil
diff --git a/mapper/mapper_test.go b/mapper/mapper_test.go
index c561663..ed36cd3 100644
--- a/mapper/mapper_test.go
+++ b/mapper/mapper_test.go
@@ -120,7 +120,7 @@
"rewrites": [
{
"@type": "koral:rewrite",
- "editor": "termMapper",
+ "editor": "Koral-Mapper",
"original": {
"@type": "koral:term",
"foundry": "opennlp",
@@ -176,7 +176,7 @@
"rewrites": [
{
"@type": "koral:rewrite",
- "editor": "termMapper",
+ "editor": "Koral-Mapper",
"original": {
"@type": "koral:term",
"foundry": "opennlp",
diff --git a/mapper/query.go b/mapper/query.go
index 73bc880..4980f85 100644
--- a/mapper/query.go
+++ b/mapper/query.go
@@ -16,8 +16,12 @@
return nil, fmt.Errorf("mapping list with ID %s not found", mappingID)
}
+ if m.mappingLists[mappingID].IsCorpus() {
+ return m.applyCorpusQueryMappings(mappingID, opts, jsonData)
+ }
+
// Get the parsed rules
- rules := m.parsedRules[mappingID]
+ rules := m.parsedQueryRules[mappingID]
// Check if we have a wrapper object with a "query" field
var queryData any
@@ -184,63 +188,7 @@
// Add rewrites if enabled and node was changed
if opts.AddRewrites && !ast.NodesEqual(node, originalNode) {
- // Create rewrite object
- rewrite := map[string]any{
- "@type": "koral:rewrite",
- "editor": "termMapper",
- }
-
- // Check if the node types are different (structural change)
- if originalNode.Type() != node.Type() {
- // Full node replacement
- originalBytes, err := parser.SerializeToJSON(originalNode)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize original node for rewrite: %w", err)
- }
- var originalJSON any
- if err := json.Unmarshal(originalBytes, &originalJSON); err != nil {
- return nil, fmt.Errorf("failed to parse original node JSON for rewrite: %w", err)
- }
- rewrite["original"] = originalJSON
- } else if term, ok := originalNode.(*ast.Term); ok && ast.IsTermNode(node) {
- // Check which attributes changed
- newTerm := node.(*ast.Term)
- if term.Foundry != newTerm.Foundry {
- rewrite["scope"] = "foundry"
- rewrite["original"] = term.Foundry
- } else if term.Layer != newTerm.Layer {
- rewrite["scope"] = "layer"
- rewrite["original"] = term.Layer
- } else if term.Key != newTerm.Key {
- rewrite["scope"] = "key"
- rewrite["original"] = term.Key
- } else if term.Value != newTerm.Value {
- rewrite["scope"] = "value"
- rewrite["original"] = term.Value
- } else {
- // No specific attribute changed, use full node replacement
- originalBytes, err := parser.SerializeToJSON(originalNode)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize original node for rewrite: %w", err)
- }
- var originalJSON any
- if err := json.Unmarshal(originalBytes, &originalJSON); err != nil {
- return nil, fmt.Errorf("failed to parse original node JSON for rewrite: %w", err)
- }
- rewrite["original"] = originalJSON
- }
- } else {
- // Full node replacement
- originalBytes, err := parser.SerializeToJSON(originalNode)
- if err != nil {
- return nil, fmt.Errorf("failed to serialize original node for rewrite: %w", err)
- }
- var originalJSON any
- if err := json.Unmarshal(originalBytes, &originalJSON); err != nil {
- return nil, fmt.Errorf("failed to parse original node JSON for rewrite: %w", err)
- }
- rewrite["original"] = originalJSON
- }
+ rewrite := buildQueryRewrite(originalNode, node)
// Add rewrite to the node
if resultMap, ok := resultData.(map[string]any); ok {
@@ -306,6 +254,36 @@
return resultData, nil
}
+// buildQueryRewrite creates a rewrite entry for a query-level transformation
+// by comparing the original and new AST nodes.
+func buildQueryRewrite(originalNode, newNode ast.Node) map[string]any {
+ if term, ok := originalNode.(*ast.Term); ok && ast.IsTermNode(newNode) && originalNode.Type() == newNode.Type() {
+ newTerm := newNode.(*ast.Term)
+ if term.Foundry != newTerm.Foundry {
+ return newRewriteEntry("foundry", term.Foundry)
+ }
+ if term.Layer != newTerm.Layer {
+ return newRewriteEntry("layer", term.Layer)
+ }
+ if term.Key != newTerm.Key {
+ return newRewriteEntry("key", term.Key)
+ }
+ if term.Value != newTerm.Value {
+ return newRewriteEntry("value", term.Value)
+ }
+ }
+
+ originalBytes, err := parser.SerializeToJSON(originalNode)
+ if err != nil {
+ return newRewriteEntry("", nil)
+ }
+ var originalJSON any
+ if err := json.Unmarshal(originalBytes, &originalJSON); err != nil {
+ return newRewriteEntry("", nil)
+ }
+ return newRewriteEntry("", originalJSON)
+}
+
// isValidQueryObject checks if the query data is a valid object that can be processed
func isValidQueryObject(data any) bool {
// Check if it's a map
diff --git a/mapper/response.go b/mapper/response.go
index d756edc..ced97f3 100644
--- a/mapper/response.go
+++ b/mapper/response.go
@@ -17,8 +17,12 @@
return nil, fmt.Errorf("mapping list with ID %s not found", mappingID)
}
+ if m.mappingLists[mappingID].IsCorpus() {
+ return m.applyCorpusResponseMappings(mappingID, opts, jsonData)
+ }
+
// Get the parsed rules
- rules := m.parsedRules[mappingID]
+ rules := m.parsedQueryRules[mappingID]
// Check if we have a snippet to process
jsonMap, ok := jsonData.(map[string]any)
diff --git a/mapper/response_test.go b/mapper/response_test.go
index dadba6c..88273c5 100644
--- a/mapper/response_test.go
+++ b/mapper/response_test.go
@@ -74,7 +74,7 @@
require.NoError(t, err)
// Debug: Print what the parsed rules look like
- rules := m.parsedRules["test-mapper"]
+ rules := m.parsedQueryRules["test-mapper"]
t.Logf("Number of parsed rules: %d", len(rules))
for i, rule := range rules {
t.Logf("Rule %d - Upper: %+v", i, rule.Upper)
diff --git a/parser/corpus_parser.go b/parser/corpus_parser.go
new file mode 100644
index 0000000..b4411ce
--- /dev/null
+++ b/parser/corpus_parser.go
@@ -0,0 +1,268 @@
+package parser
+
+import (
+ "fmt"
+ "strings"
+)
+
+// CorpusNode represents a node in a corpus mapping rule.
+type CorpusNode interface {
+ isCorpusNode()
+ Clone() CorpusNode
+ ToJSON() map[string]any
+}
+
+// CorpusField represents a single koral:doc field constraint.
+type CorpusField struct {
+ Key string
+ Value string
+ Match string // "eq","ne","geq","leq","contains","excludes" (empty = unspecified)
+ Type string // "string","regex","date" (empty = unspecified, defaults to "string")
+}
+
+func (f *CorpusField) isCorpusNode() {}
+
+func (f *CorpusField) Clone() CorpusNode {
+ return &CorpusField{Key: f.Key, Value: f.Value, Match: f.Match, Type: f.Type}
+}
+
+// ToJSON converts the field to a koral:doc JSON map.
+func (f *CorpusField) ToJSON() map[string]any {
+ m := map[string]any{
+ "@type": "koral:doc",
+ "key": f.Key,
+ "value": f.Value,
+ }
+ if f.Match != "" {
+ m["match"] = "match:" + f.Match
+ } else {
+ m["match"] = "match:eq"
+ }
+ if f.Type != "" {
+ m["type"] = "type:" + f.Type
+ } else {
+ m["type"] = "type:string"
+ }
+ return m
+}
+
+// CorpusGroup represents a koral:docGroup boolean group.
+type CorpusGroup struct {
+ Operation string // "and" or "or"
+ Operands []CorpusNode
+}
+
+func (g *CorpusGroup) isCorpusNode() {}
+
+func (g *CorpusGroup) Clone() CorpusNode {
+ ops := make([]CorpusNode, len(g.Operands))
+ for i, op := range g.Operands {
+ ops[i] = op.Clone()
+ }
+ return &CorpusGroup{Operation: g.Operation, Operands: ops}
+}
+
+// ToJSON converts the group to a koral:docGroup JSON map.
+func (g *CorpusGroup) ToJSON() map[string]any {
+ operands := make([]map[string]any, len(g.Operands))
+ for i, op := range g.Operands {
+ operands[i] = op.ToJSON()
+ }
+ return map[string]any{
+ "@type": "koral:docGroup",
+ "operation": "operation:" + g.Operation,
+ "operands": operands,
+ }
+}
+
+// CorpusMappingResult represents a parsed corpus mapping rule.
+type CorpusMappingResult struct {
+ Upper CorpusNode // Side A
+ Lower CorpusNode // Side B
+}
+
+// CorpusParser parses corpus mapping rules.
+type CorpusParser struct{}
+
+func NewCorpusParser() *CorpusParser {
+ return &CorpusParser{}
+}
+
+// ParseMapping parses a corpus mapping rule of the form "pattern <> replacement".
+func (p *CorpusParser) ParseMapping(input string) (*CorpusMappingResult, error) {
+ sepIdx := strings.Index(input, "<>")
+ if sepIdx == -1 {
+ return nil, fmt.Errorf("invalid corpus mapping rule: missing <> separator in %q", input)
+ }
+
+ leftStr := strings.TrimSpace(input[:sepIdx])
+ rightStr := strings.TrimSpace(input[sepIdx+2:])
+
+ if leftStr == "" {
+ return nil, fmt.Errorf("invalid corpus mapping rule: empty left side")
+ }
+ if rightStr == "" {
+ return nil, fmt.Errorf("invalid corpus mapping rule: empty right side")
+ }
+
+ upper, err := p.parseExpression(leftStr)
+ if err != nil {
+ return nil, fmt.Errorf("error parsing left side: %w", err)
+ }
+
+ lower, err := p.parseExpression(rightStr)
+ if err != nil {
+ return nil, fmt.Errorf("error parsing right side: %w", err)
+ }
+
+ return &CorpusMappingResult{Upper: upper, Lower: lower}, nil
+}
+
+// parseExpression parses a corpus expression (field or group).
+func (p *CorpusParser) parseExpression(input string) (CorpusNode, error) {
+ input = strings.TrimSpace(input)
+ if input == "" {
+ return nil, fmt.Errorf("empty expression")
+ }
+
+ if input[0] == '(' {
+ closeIdx := findMatchingParen(input)
+ if closeIdx == len(input)-1 {
+ return p.parseGroupContent(input[1 : len(input)-1])
+ }
+ }
+
+ return p.parseField(input)
+}
+
+// parseGroupContent parses the content inside parentheses.
+func (p *CorpusParser) parseGroupContent(input string) (CorpusNode, error) {
+ input = strings.TrimSpace(input)
+ if input == "" {
+ return nil, fmt.Errorf("empty group")
+ }
+
+ parts, operator, err := splitOnTopLevelOperator(input)
+ if err != nil {
+ return nil, err
+ }
+
+ if len(parts) == 1 {
+ return p.parseExpression(parts[0])
+ }
+
+ operands := make([]CorpusNode, len(parts))
+ for i, part := range parts {
+ node, err := p.parseExpression(strings.TrimSpace(part))
+ if err != nil {
+ return nil, fmt.Errorf("error in group operand %d: %w", i, err)
+ }
+ operands[i] = node
+ }
+
+ op := "and"
+ if operator == "|" {
+ op = "or"
+ }
+
+ return &CorpusGroup{Operation: op, Operands: operands}, nil
+}
+
+var validMatchTypes = map[string]bool{
+ "eq": true, "ne": true, "geq": true, "leq": true,
+ "contains": true, "excludes": true,
+}
+
+// parseField parses a single field expression: key=value[:match][#type].
+func (p *CorpusParser) parseField(input string) (*CorpusField, error) {
+ input = strings.TrimSpace(input)
+
+ eqIdx := strings.Index(input, "=")
+ if eqIdx == -1 {
+ return nil, fmt.Errorf("invalid field expression: missing '=' in %q", input)
+ }
+
+ key := strings.TrimSpace(input[:eqIdx])
+ rest := strings.TrimSpace(input[eqIdx+1:])
+
+ if key == "" {
+ return nil, fmt.Errorf("invalid field expression: empty key")
+ }
+ if rest == "" {
+ return nil, fmt.Errorf("invalid field expression: empty value for key %q", key)
+ }
+
+ field := &CorpusField{Key: key}
+
+ // Split off #type first
+ if hashIdx := strings.LastIndex(rest, "#"); hashIdx != -1 {
+ field.Type = strings.TrimSpace(rest[hashIdx+1:])
+ rest = rest[:hashIdx]
+ }
+
+ // Split off :match — only if the part after the last colon is a valid match type
+ if colonIdx := strings.LastIndex(rest, ":"); colonIdx != -1 {
+ candidate := strings.TrimSpace(rest[colonIdx+1:])
+ if validMatchTypes[candidate] {
+ field.Match = candidate
+ rest = rest[:colonIdx]
+ }
+ }
+
+ field.Value = strings.TrimSpace(rest)
+ if field.Value == "" {
+ return nil, fmt.Errorf("invalid field expression: empty value for key %q", key)
+ }
+
+ return field, nil
+}
+
+// findMatchingParen finds the index of the closing parenthesis matching the
+// opening parenthesis at position 0.
+func findMatchingParen(input string) int {
+ depth := 0
+ for i, ch := range input {
+ switch ch {
+ case '(':
+ depth++
+ case ')':
+ depth--
+ if depth == 0 {
+ return i
+ }
+ }
+ }
+ return -1
+}
+
+// splitOnTopLevelOperator splits a string on & or | operators at the top level
+// (not inside parentheses). Returns the parts, the operator used, and any error.
+func splitOnTopLevelOperator(input string) ([]string, string, error) {
+ depth := 0
+ var parts []string
+ var operator string
+ lastSplit := 0
+
+ for i := 0; i < len(input); i++ {
+ switch input[i] {
+ case '(':
+ depth++
+ case ')':
+ depth--
+ case '&', '|':
+ if depth == 0 {
+ op := string(input[i])
+ if operator == "" {
+ operator = op
+ } else if op != operator {
+ return nil, "", fmt.Errorf("mixed operators '&' and '|' at same level; use parentheses to disambiguate")
+ }
+ parts = append(parts, strings.TrimSpace(input[lastSplit:i]))
+ lastSplit = i + 1
+ }
+ }
+ }
+
+ parts = append(parts, strings.TrimSpace(input[lastSplit:]))
+ return parts, operator, nil
+}
diff --git a/parser/corpus_parser_test.go b/parser/corpus_parser_test.go
new file mode 100644
index 0000000..a62d0fb
--- /dev/null
+++ b/parser/corpus_parser_test.go
@@ -0,0 +1,285 @@
+package parser
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestCorpusParserSimpleField(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("textClass=novel <> genre=fiction")
+ require.NoError(t, err)
+
+ upper, ok := result.Upper.(*CorpusField)
+ require.True(t, ok)
+ assert.Equal(t, "textClass", upper.Key)
+ assert.Equal(t, "novel", upper.Value)
+ assert.Equal(t, "", upper.Match)
+ assert.Equal(t, "", upper.Type)
+
+ lower, ok := result.Lower.(*CorpusField)
+ require.True(t, ok)
+ assert.Equal(t, "genre", lower.Key)
+ assert.Equal(t, "fiction", lower.Value)
+}
+
+func TestCorpusParserMatchType(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("pubDate=2020:geq <> yearFrom=2020:geq")
+ require.NoError(t, err)
+
+ upper := result.Upper.(*CorpusField)
+ assert.Equal(t, "pubDate", upper.Key)
+ assert.Equal(t, "2020", upper.Value)
+ assert.Equal(t, "geq", upper.Match)
+
+ lower := result.Lower.(*CorpusField)
+ assert.Equal(t, "yearFrom", lower.Key)
+ assert.Equal(t, "2020", lower.Value)
+ assert.Equal(t, "geq", lower.Match)
+}
+
+func TestCorpusParserValueType(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("pubDate=2020-01#date <> year=2020#string")
+ require.NoError(t, err)
+
+ upper := result.Upper.(*CorpusField)
+ assert.Equal(t, "pubDate", upper.Key)
+ assert.Equal(t, "2020-01", upper.Value)
+ assert.Equal(t, "", upper.Match)
+ assert.Equal(t, "date", upper.Type)
+
+ lower := result.Lower.(*CorpusField)
+ assert.Equal(t, "year", lower.Key)
+ assert.Equal(t, "2020", lower.Value)
+ assert.Equal(t, "", lower.Match)
+ assert.Equal(t, "string", lower.Type)
+}
+
+func TestCorpusParserMatchAndType(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("pubDate=2020:geq#date <> year=2020:geq#string")
+ require.NoError(t, err)
+
+ upper := result.Upper.(*CorpusField)
+ assert.Equal(t, "pubDate", upper.Key)
+ assert.Equal(t, "2020", upper.Value)
+ assert.Equal(t, "geq", upper.Match)
+ assert.Equal(t, "date", upper.Type)
+
+ lower := result.Lower.(*CorpusField)
+ assert.Equal(t, "year", lower.Key)
+ assert.Equal(t, "2020", lower.Value)
+ assert.Equal(t, "geq", lower.Match)
+ assert.Equal(t, "string", lower.Type)
+}
+
+func TestCorpusParserRegex(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("textClass=wissenschaft.*#regex <> genre=science")
+ require.NoError(t, err)
+
+ upper := result.Upper.(*CorpusField)
+ assert.Equal(t, "textClass", upper.Key)
+ assert.Equal(t, "wissenschaft.*", upper.Value)
+ assert.Equal(t, "regex", upper.Type)
+
+ lower := result.Lower.(*CorpusField)
+ assert.Equal(t, "genre", lower.Key)
+ assert.Equal(t, "science", lower.Value)
+}
+
+func TestCorpusParserANDGroup(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("(textClass=novel & pubDate=2020) <> genre=fiction")
+ require.NoError(t, err)
+
+ group, ok := result.Upper.(*CorpusGroup)
+ require.True(t, ok)
+ assert.Equal(t, "and", group.Operation)
+ require.Len(t, group.Operands, 2)
+
+ f1 := group.Operands[0].(*CorpusField)
+ assert.Equal(t, "textClass", f1.Key)
+ assert.Equal(t, "novel", f1.Value)
+
+ f2 := group.Operands[1].(*CorpusField)
+ assert.Equal(t, "pubDate", f2.Key)
+ assert.Equal(t, "2020", f2.Value)
+
+ lower := result.Lower.(*CorpusField)
+ assert.Equal(t, "genre", lower.Key)
+ assert.Equal(t, "fiction", lower.Value)
+}
+
+func TestCorpusParserORGroup(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("(textClass=novel | textClass=fiction) <> genre=fiction")
+ require.NoError(t, err)
+
+ group, ok := result.Upper.(*CorpusGroup)
+ require.True(t, ok)
+ assert.Equal(t, "or", group.Operation)
+ require.Len(t, group.Operands, 2)
+
+ f1 := group.Operands[0].(*CorpusField)
+ assert.Equal(t, "textClass", f1.Key)
+ assert.Equal(t, "novel", f1.Value)
+
+ f2 := group.Operands[1].(*CorpusField)
+ assert.Equal(t, "textClass", f2.Key)
+ assert.Equal(t, "fiction", f2.Value)
+}
+
+func TestCorpusParserNestedGroup(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("(a=1 & (b=2 | c=3)) <> d=4")
+ require.NoError(t, err)
+
+ outer, ok := result.Upper.(*CorpusGroup)
+ require.True(t, ok)
+ assert.Equal(t, "and", outer.Operation)
+ require.Len(t, outer.Operands, 2)
+
+ f1 := outer.Operands[0].(*CorpusField)
+ assert.Equal(t, "a", f1.Key)
+ assert.Equal(t, "1", f1.Value)
+
+ inner, ok := outer.Operands[1].(*CorpusGroup)
+ require.True(t, ok)
+ assert.Equal(t, "or", inner.Operation)
+ require.Len(t, inner.Operands, 2)
+
+ f2 := inner.Operands[0].(*CorpusField)
+ assert.Equal(t, "b", f2.Key)
+ f3 := inner.Operands[1].(*CorpusField)
+ assert.Equal(t, "c", f3.Key)
+}
+
+func TestCorpusParserSingleToGroup(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("textClass=novel <> (genre=fiction & type=book)")
+ require.NoError(t, err)
+
+ _, ok := result.Upper.(*CorpusField)
+ require.True(t, ok)
+
+ group, ok := result.Lower.(*CorpusGroup)
+ require.True(t, ok)
+ assert.Equal(t, "and", group.Operation)
+ require.Len(t, group.Operands, 2)
+}
+
+func TestCorpusParserGroupToSingle(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("(genre=fiction & type=book) <> textClass=novel")
+ require.NoError(t, err)
+
+ _, ok := result.Upper.(*CorpusGroup)
+ require.True(t, ok)
+
+ _, ok = result.Lower.(*CorpusField)
+ require.True(t, ok)
+}
+
+func TestCorpusParserErrors(t *testing.T) {
+ p := NewCorpusParser()
+
+ _, err := p.ParseMapping("textClass=novel")
+ assert.Error(t, err, "missing <> separator")
+
+ _, err = p.ParseMapping(" <> genre=fiction")
+ assert.Error(t, err, "empty left side")
+
+ _, err = p.ParseMapping("textClass=novel <> ")
+ assert.Error(t, err, "empty right side")
+
+ _, err = p.ParseMapping("invalidfield <> genre=fiction")
+ assert.Error(t, err, "missing = in field")
+}
+
+func TestCorpusParserThreeOperandGroup(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping("(a=1 & b=2 & c=3) <> d=4")
+ require.NoError(t, err)
+
+ group, ok := result.Upper.(*CorpusGroup)
+ require.True(t, ok)
+ assert.Equal(t, "and", group.Operation)
+ require.Len(t, group.Operands, 3)
+}
+
+func TestCorpusParserWhitespaceHandling(t *testing.T) {
+ p := NewCorpusParser()
+ result, err := p.ParseMapping(" textClass=novel <> genre=fiction ")
+ require.NoError(t, err)
+
+ upper := result.Upper.(*CorpusField)
+ assert.Equal(t, "textClass", upper.Key)
+ assert.Equal(t, "novel", upper.Value)
+}
+
+func TestCorpusFieldToJSON(t *testing.T) {
+ field := &CorpusField{Key: "textClass", Value: "novel"}
+ json := field.ToJSON()
+ assert.Equal(t, "koral:doc", json["@type"])
+ assert.Equal(t, "textClass", json["key"])
+ assert.Equal(t, "novel", json["value"])
+ assert.Equal(t, "match:eq", json["match"])
+ assert.Equal(t, "type:string", json["type"])
+}
+
+func TestCorpusFieldToJSONWithMatchAndType(t *testing.T) {
+ field := &CorpusField{Key: "pubDate", Value: "2020", Match: "geq", Type: "date"}
+ json := field.ToJSON()
+ assert.Equal(t, "koral:doc", json["@type"])
+ assert.Equal(t, "pubDate", json["key"])
+ assert.Equal(t, "2020", json["value"])
+ assert.Equal(t, "match:geq", json["match"])
+ assert.Equal(t, "type:date", json["type"])
+}
+
+func TestCorpusGroupToJSON(t *testing.T) {
+ group := &CorpusGroup{
+ Operation: "and",
+ Operands: []CorpusNode{
+ &CorpusField{Key: "genre", Value: "fiction"},
+ &CorpusField{Key: "type", Value: "book"},
+ },
+ }
+ json := group.ToJSON()
+ assert.Equal(t, "koral:docGroup", json["@type"])
+ assert.Equal(t, "operation:and", json["operation"])
+ operands, ok := json["operands"].([]map[string]any)
+ require.True(t, ok)
+ require.Len(t, operands, 2)
+ assert.Equal(t, "genre", operands[0]["key"])
+ assert.Equal(t, "type", operands[1]["key"])
+}
+
+func TestCorpusFieldClone(t *testing.T) {
+ f := &CorpusField{Key: "a", Value: "b", Match: "eq", Type: "string"}
+ c := f.Clone().(*CorpusField)
+ assert.Equal(t, f.Key, c.Key)
+ assert.Equal(t, f.Value, c.Value)
+ c.Key = "changed"
+ assert.NotEqual(t, f.Key, c.Key)
+}
+
+func TestCorpusGroupClone(t *testing.T) {
+ g := &CorpusGroup{
+ Operation: "and",
+ Operands: []CorpusNode{
+ &CorpusField{Key: "a", Value: "1"},
+ &CorpusField{Key: "b", Value: "2"},
+ },
+ }
+ c := g.Clone().(*CorpusGroup)
+ assert.Equal(t, g.Operation, c.Operation)
+ require.Len(t, c.Operands, 2)
+ c.Operands[0].(*CorpusField).Key = "changed"
+ assert.NotEqual(t, g.Operands[0].(*CorpusField).Key, "changed")
+}