Added web server for transformations
diff --git a/.gitignore b/.gitignore
index f52e12c..2c6f9ce 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,4 +1,4 @@
testdata/sandbox
-cmd/termmapper
examples/
-README.md
\ No newline at end of file
+README.md
+/termmapper
\ No newline at end of file
diff --git a/cmd/termmapper/main.go b/cmd/termmapper/main.go
new file mode 100644
index 0000000..aa4e3e2
--- /dev/null
+++ b/cmd/termmapper/main.go
@@ -0,0 +1,164 @@
+package main
+
+import (
+ "flag"
+ "fmt"
+ "os"
+ "os/signal"
+ "strings"
+ "syscall"
+
+ "github.com/KorAP/KoralPipe-TermMapper2/pkg/mapper"
+ "github.com/gofiber/fiber/v2"
+ "github.com/rs/zerolog"
+ "github.com/rs/zerolog/log"
+)
+
+type config struct {
+ port int
+ config string
+ logLevel string
+}
+
+func parseFlags() *config {
+ cfg := &config{}
+
+ flag.IntVar(&cfg.port, "port", 8080, "Port to listen on")
+ flag.IntVar(&cfg.port, "p", 8080, "Port to listen on (shorthand)")
+
+ flag.StringVar(&cfg.config, "config", "", "YAML configuration file containing mapping directives")
+ flag.StringVar(&cfg.config, "c", "", "YAML configuration file containing mapping directives (shorthand)")
+
+ flag.StringVar(&cfg.logLevel, "log-level", "info", "Log level (debug, info, warn, error)")
+ flag.StringVar(&cfg.logLevel, "l", "info", "Log level (shorthand)")
+
+ flag.Usage = func() {
+ fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
+ fmt.Fprintf(os.Stderr, "\nA web service for transforming JSON objects using term mapping rules.\n\n")
+ fmt.Fprintf(os.Stderr, "Options:\n")
+ flag.PrintDefaults()
+ }
+
+ flag.Parse()
+
+ if cfg.config == "" {
+ fmt.Fprintln(os.Stderr, "Error: config file is required")
+ flag.Usage()
+ os.Exit(1)
+ }
+
+ return cfg
+}
+
+func setupLogger(level string) {
+ // Parse log level
+ lvl, err := zerolog.ParseLevel(strings.ToLower(level))
+ if err != nil {
+ log.Error().Err(err).Str("level", level).Msg("Invalid log level, defaulting to info")
+ lvl = zerolog.InfoLevel
+ }
+
+ // Configure zerolog
+ zerolog.SetGlobalLevel(lvl)
+ log.Logger = log.Output(zerolog.ConsoleWriter{Out: os.Stderr})
+}
+
+func main() {
+ // Parse command line flags
+ cfg := parseFlags()
+
+ // Set up logging
+ setupLogger(cfg.logLevel)
+
+ // Create a new mapper instance
+ m, err := mapper.NewMapper(cfg.config)
+ if err != nil {
+ log.Fatal().Err(err).Msg("Failed to create mapper")
+ }
+
+ // Create fiber app
+ app := fiber.New(fiber.Config{
+ DisableStartupMessage: true,
+ })
+
+ // Set up routes
+ setupRoutes(app, m)
+
+ // Start server
+ go func() {
+ log.Info().Int("port", cfg.port).Msg("Starting server")
+ if err := app.Listen(fmt.Sprintf(":%d", cfg.port)); err != nil {
+ log.Fatal().Err(err).Msg("Server error")
+ }
+ }()
+
+ // Wait for interrupt signal
+ sigChan := make(chan os.Signal, 1)
+ signal.Notify(sigChan, os.Interrupt, syscall.SIGTERM)
+ <-sigChan
+
+ // Graceful shutdown
+ log.Info().Msg("Shutting down server")
+ if err := app.Shutdown(); err != nil {
+ log.Error().Err(err).Msg("Error during shutdown")
+ }
+}
+
+func setupRoutes(app *fiber.App, m *mapper.Mapper) {
+ // Health check endpoint
+ app.Get("/health", func(c *fiber.Ctx) error {
+ return c.SendString("OK")
+ })
+
+ // Transformation endpoint
+ app.Post("/:map/query", handleTransform(m))
+}
+
+func handleTransform(m *mapper.Mapper) fiber.Handler {
+ return func(c *fiber.Ctx) error {
+ // Get parameters
+ mapID := c.Params("map")
+ dir := c.Query("dir", "atob")
+ foundryA := c.Query("foundryA", "")
+ foundryB := c.Query("foundryB", "")
+ layerA := c.Query("layerA", "")
+ layerB := c.Query("layerB", "")
+
+ // Validate direction
+ if dir != "atob" && dir != "btoa" {
+ return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
+ "error": "invalid direction, must be 'atob' or 'btoa'",
+ })
+ }
+
+ // Parse request body
+ var jsonData interface{}
+ if err := c.BodyParser(&jsonData); err != nil {
+ return c.Status(fiber.StatusBadRequest).JSON(fiber.Map{
+ "error": "invalid JSON in request body",
+ })
+ }
+
+ // Apply mappings
+ result, err := m.ApplyMappings(mapID, mapper.MappingOptions{
+ Direction: mapper.Direction(dir),
+ FoundryA: foundryA,
+ FoundryB: foundryB,
+ LayerA: layerA,
+ LayerB: layerB,
+ }, jsonData)
+
+ if err != nil {
+ log.Error().Err(err).
+ Str("mapID", mapID).
+ Str("direction", dir).
+ Msg("Failed to apply mappings")
+
+ return c.Status(fiber.StatusInternalServerError).JSON(fiber.Map{
+ "error": err.Error(),
+ })
+ }
+
+ return c.JSON(result)
+ }
+}
diff --git a/cmd/termmapper/main_test.go b/cmd/termmapper/main_test.go
new file mode 100644
index 0000000..0387cc6
--- /dev/null
+++ b/cmd/termmapper/main_test.go
@@ -0,0 +1,286 @@
+package main
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/KorAP/KoralPipe-TermMapper2/pkg/mapper"
+ "github.com/gofiber/fiber/v2"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestTransformEndpoint(t *testing.T) {
+ // Create a temporary config file
+ tmpDir := t.TempDir()
+ configFile := filepath.Join(tmpDir, "test-config.yaml")
+
+ configContent := `- id: test-mapper
+ foundryA: opennlp
+ layerA: p
+ foundryB: upos
+ layerB: p
+ mappings:
+ - "[PIDAT] <> [opennlp/p=PIDAT & opennlp/p=AdjType:Pdt]"
+ - "[DET] <> [opennlp/p=DET]"`
+
+ err := os.WriteFile(configFile, []byte(configContent), 0644)
+ require.NoError(t, err)
+
+ // Create mapper
+ m, err := mapper.NewMapper(configFile)
+ require.NoError(t, err)
+
+ // Create fiber app
+ app := fiber.New()
+ setupRoutes(app, m)
+
+ tests := []struct {
+ name string
+ mapID string
+ direction string
+ foundryA string
+ foundryB string
+ layerA string
+ layerB string
+ input string
+ expectedCode int
+ expectedBody string
+ expectedError string
+ }{
+ {
+ name: "Simple A to B mapping",
+ mapID: "test-mapper",
+ direction: "atob",
+ input: `{
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "PIDAT",
+ "layer": "p",
+ "match": "match:eq"
+ }
+ }`,
+ expectedCode: http.StatusOK,
+ expectedBody: `{
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:termGroup",
+ "operands": [
+ {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "PIDAT",
+ "layer": "p",
+ "match": "match:eq"
+ },
+ {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "AdjType",
+ "layer": "p",
+ "match": "match:eq",
+ "value": "Pdt"
+ }
+ ],
+ "relation": "relation:and"
+ }
+ }`,
+ },
+ {
+ name: "B to A mapping",
+ mapID: "test-mapper",
+ direction: "btoa",
+ input: `{
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:termGroup",
+ "operands": [
+ {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "PIDAT",
+ "layer": "p",
+ "match": "match:eq"
+ },
+ {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "AdjType",
+ "layer": "p",
+ "match": "match:eq",
+ "value": "Pdt"
+ }
+ ],
+ "relation": "relation:and"
+ }
+ }`,
+ expectedCode: http.StatusOK,
+ expectedBody: `{
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "PIDAT",
+ "layer": "p",
+ "match": "match:eq"
+ }
+ }`,
+ },
+ {
+ name: "Mapping with foundry override",
+ mapID: "test-mapper",
+ direction: "atob",
+ foundryB: "custom",
+ input: `{
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:term",
+ "foundry": "opennlp",
+ "key": "PIDAT",
+ "layer": "p",
+ "match": "match:eq"
+ }
+ }`,
+ expectedCode: http.StatusOK,
+ expectedBody: `{
+ "@type": "koral:token",
+ "wrap": {
+ "@type": "koral:termGroup",
+ "operands": [
+ {
+ "@type": "koral:term",
+ "foundry": "custom",
+ "key": "PIDAT",
+ "layer": "p",
+ "match": "match:eq"
+ },
+ {
+ "@type": "koral:term",
+ "foundry": "custom",
+ "key": "AdjType",
+ "layer": "p",
+ "match": "match:eq",
+ "value": "Pdt"
+ }
+ ],
+ "relation": "relation:and"
+ }
+ }`,
+ },
+ {
+ name: "Invalid mapping ID",
+ mapID: "nonexistent",
+ direction: "atob",
+ input: `{"@type": "koral:token"}`,
+ expectedCode: http.StatusInternalServerError,
+ expectedError: "mapping list with ID nonexistent not found",
+ },
+ {
+ name: "Invalid direction",
+ mapID: "test-mapper",
+ direction: "invalid",
+ input: `{"@type": "koral:token"}`,
+ expectedCode: http.StatusBadRequest,
+ expectedError: "invalid direction, must be 'atob' or 'btoa'",
+ },
+ {
+ name: "Invalid JSON",
+ mapID: "test-mapper",
+ direction: "atob",
+ input: `invalid json`,
+ expectedCode: http.StatusBadRequest,
+ expectedError: "invalid JSON in request body",
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ // Build URL with query parameters
+ url := "/" + tt.mapID + "/query"
+ if tt.direction != "" {
+ url += "?dir=" + tt.direction
+ }
+ if tt.foundryA != "" {
+ url += "&foundryA=" + tt.foundryA
+ }
+ if tt.foundryB != "" {
+ url += "&foundryB=" + tt.foundryB
+ }
+ if tt.layerA != "" {
+ url += "&layerA=" + tt.layerA
+ }
+ if tt.layerB != "" {
+ url += "&layerB=" + tt.layerB
+ }
+
+ // Make request
+ req := httptest.NewRequest(http.MethodPost, url, bytes.NewBufferString(tt.input))
+ req.Header.Set("Content-Type", "application/json")
+ resp, err := app.Test(req)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+
+ // Check status code
+ assert.Equal(t, tt.expectedCode, resp.StatusCode)
+
+ // Read response body
+ body, err := io.ReadAll(resp.Body)
+ require.NoError(t, err)
+
+ if tt.expectedError != "" {
+ // Check error message
+ var errResp fiber.Map
+ err = json.Unmarshal(body, &errResp)
+ require.NoError(t, err)
+ assert.Equal(t, tt.expectedError, errResp["error"])
+ } else {
+ // Compare JSON responses
+ var expected, actual interface{}
+ err = json.Unmarshal([]byte(tt.expectedBody), &expected)
+ require.NoError(t, err)
+ err = json.Unmarshal(body, &actual)
+ require.NoError(t, err)
+ assert.Equal(t, expected, actual)
+ }
+ })
+ }
+}
+
+func TestHealthEndpoint(t *testing.T) {
+ // Create a temporary config file for the mapper
+ tmpDir := t.TempDir()
+ configFile := filepath.Join(tmpDir, "test-config.yaml")
+ configContent := `- id: test-mapper
+ mappings:
+ - "[A] <> [B]"`
+
+ err := os.WriteFile(configFile, []byte(configContent), 0644)
+ require.NoError(t, err)
+
+ // Create mapper with config
+ m, err := mapper.NewMapper(configFile)
+ require.NoError(t, err)
+
+ // Create fiber app
+ app := fiber.New()
+ setupRoutes(app, m)
+
+ // Test health endpoint
+ req := httptest.NewRequest(http.MethodGet, "/health", nil)
+ resp, err := app.Test(req)
+ require.NoError(t, err)
+ defer resp.Body.Close()
+
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
+ body, err := io.ReadAll(resp.Body)
+ require.NoError(t, err)
+ assert.Equal(t, "OK", string(body))
+}
diff --git a/go.mod b/go.mod
index 1ff8260..190aa82 100644
--- a/go.mod
+++ b/go.mod
@@ -1,14 +1,28 @@
module github.com/KorAP/KoralPipe-TermMapper2
-go 1.22.2
+go 1.23.0
+
+toolchain go1.23.9
require (
github.com/alecthomas/participle/v2 v2.1.4
+ github.com/gofiber/fiber/v2 v2.52.8
+ github.com/rs/zerolog v1.34.0
github.com/stretchr/testify v1.10.0
gopkg.in/yaml.v3 v3.0.1
)
require (
+ github.com/andybalholm/brotli v1.1.1 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
+ github.com/google/uuid v1.6.0 // indirect
+ github.com/klauspost/compress v1.18.0 // indirect
+ github.com/mattn/go-colorable v0.1.14 // indirect
+ github.com/mattn/go-isatty v0.0.20 // indirect
+ github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
+ github.com/rivo/uniseg v0.4.7 // indirect
+ github.com/valyala/bytebufferpool v1.0.0 // indirect
+ github.com/valyala/fasthttp v1.62.0 // indirect
+ golang.org/x/sys v0.33.0 // indirect
)
diff --git a/go.sum b/go.sum
index eb1dd50..7d1da80 100644
--- a/go.sum
+++ b/go.sum
@@ -4,14 +4,51 @@
github.com/alecthomas/participle/v2 v2.1.4/go.mod h1:8tqVbpTX20Ru4NfYQgZf4mP18eXPTBViyMWiArNEgGI=
github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc=
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
+github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA=
+github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA=
+github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofiber/fiber/v2 v2.52.8 h1:xl4jJQ0BV5EJTA2aWiKw/VddRpHrKeZLF0QPUxqn0x4=
+github.com/gofiber/fiber/v2 v2.52.8/go.mod h1:YEcBbO/FB+5M1IZNBP9FO3J9281zgPAreiI1oqg8nDw=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
+github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
+github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
+github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw=
+github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc=
+github.com/valyala/fasthttp v1.62.0 h1:8dKRBX/y2rCzyc6903Zu1+3qN0H/d2MsxPPmVNamiH0=
+github.com/valyala/fasthttp v1.62.0/go.mod h1:FCINgr4GKdKqV8Q0xv8b+UxPV+H/O5nNFo3D+r54Htg=
+github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
+github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=