From e2cbb72cc8fb446ec391b38ec69d6878ccd43169 Mon Sep 17 00:00:00 2001 From: Atomys Date: Fri, 30 Jun 2023 14:34:07 +0200 Subject: [PATCH] feat: formatting feature for postgresql query (#145) **Relative Issues:** Resolve #106 **Describe the pull request** This pull request introduces a new feature: a formatting function for PostgreSQL queries. This feature is designed to improve the readability and maintainability of SQL queries by automatically applying consistent formatting rules. The objective of this feature is to help developers write clearer, more understandable SQL queries, making the code easier to read, understand, and debug. By introducing this formatting feature, we aim to enhance developer productivity and the overall quality of our codebase. **Checklist** - [ ] I have linked the relative issue to this pull request - [ ] I have made the modifications or added tests related to my PR - [ ] I have added/updated the documentation for my RP - [ ] I put my PR in Ready for Review only when all the checklist is checked --- go.mod | 1 + go.sum | 13 ++ internal/server/v1alpha1/handlers.go | 29 ++-- pkg/formatting/format.go | 78 ----------- pkg/formatting/formatter.go | 128 ++++++++++++++++++ .../{format_test.go => formatter_test.go} | 81 ++++++----- pkg/storage/postgres/postgres.go | 81 +++++++++-- pkg/storage/postgres/postgres_test.go | 69 +++++++++- pkg/storage/rabbitmq/rabbitmq.go | 6 +- pkg/storage/rabbitmq/rabbitmq_test.go | 9 +- pkg/storage/redis/redis.go | 8 +- pkg/storage/redis/redis_test.go | 6 +- pkg/storage/storage.go | 3 +- tests/webhooks.tests.yml | 18 ++- 14 files changed, 378 insertions(+), 152 deletions(-) delete mode 100644 pkg/formatting/format.go create mode 100644 pkg/formatting/formatter.go rename pkg/formatting/{format_test.go => formatter_test.go} (61%) diff --git a/go.mod b/go.mod index 97196b9..3744a37 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jmoiron/sqlx v1.3.5 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect diff --git a/go.sum b/go.sum index 2248fcc..0f7d0a4 100644 --- a/go.sum +++ b/go.sum @@ -69,8 +69,10 @@ github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbS github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI= github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -141,8 +143,13 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jmoiron/sqlx v1.3.5 h1:vFFPA71p1o5gAeqtEAwLU4dnX2napprKtHr7PYIcN3g= +github.com/jmoiron/sqlx v1.3.5/go.mod h1:nRVWtLre0KfCLJvgxzCsLVMogSvQ1zNJtpYr2Ccp0mQ= +github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -150,6 +157,7 @@ github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= @@ -158,10 +166,14 @@ github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZb github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.18.1 h1:M1GfJqGRrBrrGGsbxzV5dqM2U2ApXefZCQpkukxYRLE= @@ -522,6 +534,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/server/v1alpha1/handlers.go b/internal/server/v1alpha1/handlers.go index e04ac02..f97f29b 100644 --- a/internal/server/v1alpha1/handlers.go +++ b/internal/server/v1alpha1/handlers.go @@ -87,6 +87,8 @@ func (s *Server) WebhookHandler() http.HandlerFunc { // it will call the security pipeline if configured and store data on each configured // storages func webhookService(s *Server, spec *config.WebhookSpec, r *http.Request) (err error) { + ctx := r.Context() + if spec == nil { return config.ErrSpecNotFound } @@ -107,21 +109,28 @@ func webhookService(s *Server, spec *config.WebhookSpec, r *http.Request) (err e } } + previousPayload := data + payloadFormatter := formatting.New(). + WithRequest(r). + WithPayload(data). + WithData("Spec", spec). + WithData("Config", config.Current()) + for _, storage := range spec.Storage { - str, err := formatting. - NewTemplateData(storage.Formatting.Template). - WithRequest(r). - WithPayload(data). - WithData("Spec", spec). - WithData("Storage", storage). - WithData("Config", config.Current()). - Render() + payloadFormatter = payloadFormatter.WithData("Storage", storage) + + storagePayload, err := payloadFormatter.WithTemplate(storage.Formatting.Template).Render() if err != nil { return err } - log.Debug().Msgf("store following data: %+v", str) - if err := storage.Client.Push(str); err != nil { + // update the formatter with the rendered payload of storage formatting + // this will allow to chain formatting + payloadFormatter.WithData("PreviousPayload", previousPayload) + ctx = formatting.ToContext(ctx, payloadFormatter) + + log.Debug().Msgf("store following data: %s", storagePayload) + if err := storage.Client.Push(ctx, []byte(storagePayload)); err != nil { return err } log.Debug().Str("storage", storage.Client.Name()).Msgf("stored successfully") diff --git a/pkg/formatting/format.go b/pkg/formatting/format.go deleted file mode 100644 index 0bd9b17..0000000 --- a/pkg/formatting/format.go +++ /dev/null @@ -1,78 +0,0 @@ -package formatting - -import ( - "bytes" - "fmt" - "net/http" - "sync" - "text/template" - - "github.com/rs/zerolog/log" -) - -type TemplateData struct { - tmplString string - - mu sync.RWMutex // protect following field amd template parsing - data map[string]interface{} -} - -// NewTemplateData returns a new TemplateData instance. It takes the template -// string as a parameter. The template string is the string that will be used -// to render the template. The data is the map of data that will be used to -// render the template. -func NewTemplateData(tmplString string) *TemplateData { - return &TemplateData{ - tmplString: tmplString, - data: make(map[string]interface{}), - mu: sync.RWMutex{}, - } -} - -// WithData adds a key-value pair to the data map. The key is the name of the -// variable and the value is the value of the variable. -func (d *TemplateData) WithData(name string, data interface{}) *TemplateData { - d.mu.Lock() - defer d.mu.Unlock() - - d.data[name] = data - return d -} - -// WithRequest adds a http.Request object to the data map. The key of request is -// "Request". -func (d *TemplateData) WithRequest(r *http.Request) *TemplateData { - d.WithData("Request", r) - return d -} - -// WithPayload adds a payload to the data map. The key of payload is "Payload". -// The payload is basically the body of the request. -func (d *TemplateData) WithPayload(payload []byte) *TemplateData { - d.WithData("Payload", string(payload)) - return d -} - -// Render returns the rendered template string. It takes the template string -// from the TemplateData instance and the data stored in the TemplateData -// instance. It returns an error if the template string is invalid or when -// rendering the template fails. -func (d *TemplateData) Render() (string, error) { - d.mu.RLock() - defer d.mu.RUnlock() - - log.Debug().Msgf("rendering template: %s", d.tmplString) - - t := template.New("formattingTmpl").Funcs(funcMap()) - t, err := t.Parse(d.tmplString) - if err != nil { - return "", fmt.Errorf("error in your template: %s", err.Error()) - } - - buf := new(bytes.Buffer) - if err := t.Execute(buf, d.data); err != nil { - return "", fmt.Errorf("error while filling your template: %s", err.Error()) - } - - return buf.String(), nil -} diff --git a/pkg/formatting/formatter.go b/pkg/formatting/formatter.go new file mode 100644 index 0000000..d32dd10 --- /dev/null +++ b/pkg/formatting/formatter.go @@ -0,0 +1,128 @@ +package formatting + +import ( + "bytes" + "context" + "fmt" + "net/http" + "sync" + "text/template" + + "github.com/rs/zerolog/log" +) + +type Formatter struct { + tmplString string + + mu sync.RWMutex // protect following field amd template parsing + data map[string]interface{} +} + +var ( + formatterCtxKey = struct{}{} + // ErrNotFoundInContext is returned when the formatting data is not found in + // the context. Use `FromContext` and `ToContext` to set and get the data in + // the context. + ErrNotFoundInContext = fmt.Errorf("unable to get the formatting data from the context") + // ErrNoTemplate is returned when no template is defined in the Formatter + // instance. Provide a template using the WithTemplate method. + ErrNoTemplate = fmt.Errorf("no template defined") +) + +// NewWithTemplate returns a new Formatter instance. It takes the template +// string as a parameter. The template string is the string that will be used +// to render the template. The data is the map of data that will be used to +// render the template. +// ! DEPRECATED: use New() and WithTemplate() instead +func NewWithTemplate(tmplString string) *Formatter { + return &Formatter{ + tmplString: tmplString, + data: make(map[string]interface{}), + mu: sync.RWMutex{}, + } +} + +// New returns a new Formatter instance. It takes no parameters. The template +// string must be set using the WithTemplate method. The data is the map of data +// that will be used to render the template. +func New() *Formatter { + return &Formatter{ + data: make(map[string]interface{}), + mu: sync.RWMutex{}, + } +} + +// WithTemplate sets the template string. The template string is the string that +// will be used to render the template. +func (d *Formatter) WithTemplate(tmplString string) *Formatter { + d.tmplString = tmplString + return d +} + +// WithData adds a key-value pair to the data map. The key is the name of the +// variable and the value is the value of the variable. +func (d *Formatter) WithData(name string, data interface{}) *Formatter { + d.mu.Lock() + defer d.mu.Unlock() + + d.data[name] = data + return d +} + +// WithRequest adds a http.Request object to the data map. The key of request is +// "Request". +func (d *Formatter) WithRequest(r *http.Request) *Formatter { + d.WithData("Request", r) + return d +} + +// WithPayload adds a payload to the data map. The key of payload is "Payload". +// The payload is basically the body of the request. +func (d *Formatter) WithPayload(payload []byte) *Formatter { + d.WithData("Payload", string(payload)) + return d +} + +// Render returns the rendered template string. It takes the template string +// from the Formatter instance and the data stored in the Formatter +// instance. It returns an error if the template string is invalid or when +// rendering the template fails. +func (d *Formatter) Render() (string, error) { + d.mu.RLock() + defer d.mu.RUnlock() + + if d.tmplString == "" { + return "", ErrNoTemplate + } + + log.Debug().Msgf("rendering template: %s", d.tmplString) + + t := template.New("formattingTmpl").Funcs(funcMap()) + t, err := t.Parse(d.tmplString) + if err != nil { + return "", fmt.Errorf("error in your template: %s", err.Error()) + } + + buf := new(bytes.Buffer) + if err := t.Execute(buf, d.data); err != nil { + return "", fmt.Errorf("error while filling your template: %s", err.Error()) + } + + return buf.String(), nil +} + +// FromContext returns the Formatter instance stored in the context. It returns +// an error if the Formatter instance is not found in the context. +func FromContext(ctx context.Context) (*Formatter, error) { + d, ok := ctx.Value(formatterCtxKey).(*Formatter) + if !ok { + return nil, ErrNotFoundInContext + } + return d, nil +} + +// ToContext adds the Formatter instance to the context. It returns the context +// with the Formatter instance. +func ToContext(ctx context.Context, d *Formatter) context.Context { + return context.WithValue(ctx, formatterCtxKey, d) +} diff --git a/pkg/formatting/format_test.go b/pkg/formatting/formatter_test.go similarity index 61% rename from pkg/formatting/format_test.go rename to pkg/formatting/formatter_test.go index f4efe95..da11a6f 100644 --- a/pkg/formatting/format_test.go +++ b/pkg/formatting/formatter_test.go @@ -1,25 +1,29 @@ package formatting import ( + "context" "encoding/json" "net/http" "net/http/httptest" "testing" "github.com/stretchr/testify/assert" - - "atomys.codes/webhooked/internal/config" ) -func TestNewTemplateData(t *testing.T) { +func TestNewWithTemplate(t *testing.T) { assert := assert.New(t) - tmpl := NewTemplateData("") + tmpl := New().WithTemplate("") assert.NotNil(tmpl) assert.Equal("", tmpl.tmplString) assert.Equal(0, len(tmpl.data)) - tmpl = NewTemplateData("{{ .Payload }}") + tmpl = New().WithTemplate("{{ .Payload }}") + assert.NotNil(tmpl) + assert.Equal("{{ .Payload }}", tmpl.tmplString) + assert.Equal(0, len(tmpl.data)) + + tmpl = NewWithTemplate("{{ .Payload }}") assert.NotNil(tmpl) assert.Equal("{{ .Payload }}", tmpl.tmplString) assert.Equal(0, len(tmpl.data)) @@ -28,7 +32,7 @@ func TestNewTemplateData(t *testing.T) { func Test_WithData(t *testing.T) { assert := assert.New(t) - tmpl := NewTemplateData("").WithData("test", true) + tmpl := New().WithTemplate("").WithData("test", true) assert.NotNil(tmpl) assert.Equal("", tmpl.tmplString) assert.Equal(1, len(tmpl.data)) @@ -38,7 +42,7 @@ func Test_WithData(t *testing.T) { func Test_WithRequest(t *testing.T) { assert := assert.New(t) - tmpl := NewTemplateData("").WithRequest(httptest.NewRequest("GET", "/", nil)) + tmpl := New().WithTemplate("").WithRequest(httptest.NewRequest("GET", "/", nil)) assert.NotNil(tmpl) assert.Equal("", tmpl.tmplString) assert.Equal(1, len(tmpl.data)) @@ -53,7 +57,7 @@ func Test_WithPayload(t *testing.T) { data, err := json.Marshal(map[string]interface{}{"test": "test"}) assert.Nil(err) - tmpl := NewTemplateData("").WithPayload(data) + tmpl := New().WithTemplate("").WithPayload(data) assert.NotNil(tmpl) assert.Equal("", tmpl.tmplString) assert.Equal(1, len(tmpl.data)) @@ -63,8 +67,12 @@ func Test_WithPayload(t *testing.T) { func Test_Render(t *testing.T) { assert := assert.New(t) + // Test with no template + _, err := New().Render() + assert.ErrorIs(err, ErrNoTemplate) + // Test with basic template - tmpl := NewTemplateData("{{ .Payload }}").WithPayload([]byte(`{"test": "test"}`)) + tmpl := New().WithTemplate("{{ .Payload }}").WithPayload([]byte(`{"test": "test"}`)) assert.NotNil(tmpl) assert.Equal("{{ .Payload }}", tmpl.tmplString) assert.Equal(1, len(tmpl.data)) @@ -79,11 +87,9 @@ func Test_Render(t *testing.T) { req := httptest.NewRequest("GET", "/", nil) req.Header.Set("X-Test", "test") - tmpl = NewTemplateData(` + tmpl = New().WithTemplate(` { - "config": {{ toJson .Config }}, - "spec": {{ toJson .Spec }}, - "storage": {{ toJson .Storage }}, + "customData": {{ toJson .CustomData }}, "metadata": { "testID": "{{ .Request.Header | getHeader "X-Test" }}", "deliveryID": "{{ .Request.Header | getHeader "X-Delivery" | default "unknown" }}" @@ -97,27 +103,14 @@ func Test_Render(t *testing.T) { `). WithPayload([]byte(`{"test": {"foo": true}}`)). WithRequest(req). - WithData("Spec", &config.WebhookSpec{Name: "test", EntrypointURL: "/webhooks/test", Formatting: &config.FormattingSpec{}}). - WithData("Storage", &config.StorageSpec{Type: "testing", Specs: map[string]interface{}{}}). - WithData("Config", config.Current()) + WithData("CustomData", map[string]string{"foo": "bar"}) assert.NotNil(tmpl) str, err = tmpl.Render() assert.Nil(err) assert.JSONEq(`{ - "config": { - "apiVersion":"", - "observability":{ - "metricsEnabled":false - }, - "specs": null - }, - "spec": { - "name":"test", - "entrypointUrl": "/webhooks/test" - }, - "storage": { - "type": "testing" + "customData": { + "foo": "bar" }, "metadata": { "testID": "test", @@ -129,7 +122,7 @@ func Test_Render(t *testing.T) { }`, str) // Test with template with template error - tmpl = NewTemplateData("{{ .Payload }") + tmpl = New().WithTemplate("{{ .Payload }") assert.NotNil(tmpl) assert.Equal("{{ .Payload }", tmpl.tmplString) @@ -139,7 +132,7 @@ func Test_Render(t *testing.T) { assert.Equal("", str) // Test with template with data error - tmpl = NewTemplateData("{{ .Request.Method }}").WithRequest(nil) + tmpl = New().WithTemplate("{{ .Request.Method }}").WithRequest(nil) assert.NotNil(tmpl) assert.Equal("{{ .Request.Method }}", tmpl.tmplString) @@ -148,3 +141,29 @@ func Test_Render(t *testing.T) { assert.Contains(err.Error(), "error while filling your template: ") assert.Equal("", str) } + +func TestFromContext(t *testing.T) { + // Test case 1: context value is not a *Formatter + ctx1 := context.Background() + _, err1 := FromContext(ctx1) + assert.Equal(t, ErrNotFoundInContext, err1) + + // Test case 2: context value is a *Formatter + ctx2 := context.WithValue(context.Background(), formatterCtxKey, &Formatter{}) + formatter, err2 := FromContext(ctx2) + assert.NotNil(t, formatter) + assert.Nil(t, err2) +} + +func TestToContext(t *testing.T) { + // Test case 1: context value is nil + ctx1 := context.Background() + ctx1 = ToContext(ctx1, nil) + assert.Nil(t, ctx1.Value(formatterCtxKey)) + + // Test case 2: context value is not nil + ctx2 := context.Background() + formatter := &Formatter{} + ctx2 = ToContext(ctx2, formatter) + assert.Equal(t, formatter, ctx2.Value(formatterCtxKey)) +} diff --git a/pkg/storage/postgres/postgres.go b/pkg/storage/postgres/postgres.go index 39b62fc..9429fcb 100644 --- a/pkg/storage/postgres/postgres.go +++ b/pkg/storage/postgres/postgres.go @@ -1,18 +1,21 @@ package postgres import ( - "database/sql" + "context" "fmt" + "github.com/jmoiron/sqlx" _ "github.com/lib/pq" + "github.com/rs/zerolog/log" "atomys.codes/webhooked/internal/valuable" + "atomys.codes/webhooked/pkg/formatting" ) // storage is the struct contains client and config // Run is made from external caller at begins programs type storage struct { - client *sql.DB + client *sqlx.DB config *config } @@ -20,8 +23,16 @@ type storage struct { // Run is made from internal caller type config struct { DatabaseURL valuable.Valuable `mapstructure:"databaseUrl" json:"databaseUrl"` - TableName string `mapstructure:"tableName" json:"tableName"` - DataField string `mapstructure:"dataField" json:"dataField"` + // ! Deprecation notice: End of life in v1.0.0 + TableName string `mapstructure:"tableName" json:"tableName"` + // ! Deprecation notice: End of life in v1.0.0 + DataField string `mapstructure:"dataField" json:"dataField"` + + UseFormattingToPerformQuery bool `mapstructure:"useFormattingToPerformQuery" json:"useFormattingToPerformQuery"` + // The query to perform on the database with named arguments + Query string `mapstructure:"query" json:"query"` + // The arguments to use in the query with the formatting feature (see pkg/formatting) + Args map[string]string `mapstructure:"args" json:"args"` } // NewStorage is the function for create new Postgres client storage @@ -40,7 +51,26 @@ func NewStorage(configRaw map[string]interface{}) (*storage, error) { return nil, err } - if newClient.client, err = sql.Open("postgres", newClient.config.DatabaseURL.First()); err != nil { + // ! Deprecation notice: End of life in v1.0.0 + if newClient.config.TableName != "" || newClient.config.DataField != "" { + log.Warn().Msg("[DEPRECATION NOTICE] The TableName and DataField are deprecated, please use the formatting feature instead") + } + + if newClient.config.UseFormattingToPerformQuery { + if newClient.config.TableName != "" || newClient.config.DataField != "" { + return nil, fmt.Errorf("the formatting feature is enabled, the TableName and DataField are deprecated and cannot be used in the same time") + } + + if newClient.config.Query == "" { + return nil, fmt.Errorf("the query is required when the formatting feature is enabled") + } + + if newClient.config.Args == nil { + newClient.config.Args = make(map[string]string, 0) + } + } + + if newClient.client, err = sqlx.Open("postgres", newClient.config.DatabaseURL.First()); err != nil { return nil, err } @@ -53,15 +83,46 @@ func (c storage) Name() string { return "postgres" } -// Push is the function for push data in the storage +// Push is the function for push data in the storage. +// The data is formatted with the formatting feature and be serialized by the +// client with "toSql" method // A run is made from external caller // @param value that will be pushed // @return an error if the push failed -func (c storage) Push(value interface{}) error { - request := fmt.Sprintf("INSERT INTO %s(%s) VALUES ($1)", c.config.TableName, c.config.DataField) - if _, err := c.client.Query(request, value); err != nil { +func (c storage) Push(ctx context.Context, value []byte) error { + // ! Deprecation notice: End of life in v1.0.0 + if !c.config.UseFormattingToPerformQuery { + request := fmt.Sprintf("INSERT INTO %s(%s) VALUES ($1)", c.config.TableName, c.config.DataField) + if _, err := c.client.Query(request, value); err != nil { + return err + } + return nil + } + + formatter, err := formatting.FromContext(ctx) + if err != nil { + return err + } + + stmt, err := c.client.PrepareNamedContext(ctx, c.config.Query) + if err != nil { return err } - return nil + var namedArgs = make(map[string]interface{}, 0) + for name, template := range c.config.Args { + value, err := formatter. + WithPayload(value). + WithTemplate(template). + WithData("FieldName", name). + Render() + if err != nil { + return err + } + + namedArgs[name] = value + } + + _, err = stmt.QueryContext(ctx, namedArgs) + return err } diff --git a/pkg/storage/postgres/postgres_test.go b/pkg/storage/postgres/postgres_test.go index 0f61427..bd4463d 100644 --- a/pkg/storage/postgres/postgres_test.go +++ b/pkg/storage/postgres/postgres_test.go @@ -1,19 +1,22 @@ package postgres import ( - "database/sql" + "context" "fmt" "os" "testing" + "atomys.codes/webhooked/pkg/formatting" + "github.com/jmoiron/sqlx" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/suite" ) type PostgresSetupTestSuite struct { suite.Suite - client *sql.DB + client *sqlx.DB databaseUrl string + ctx context.Context } // Create Table for running test @@ -29,12 +32,18 @@ func (suite *PostgresSetupTestSuite) BeforeTest(suiteName, testName string) { os.Getenv("POSTGRES_DB"), ) - if suite.client, err = sql.Open("postgres", suite.databaseUrl); err != nil { + if suite.client, err = sqlx.Open("postgres", suite.databaseUrl); err != nil { suite.T().Error(err) } if _, err := suite.client.Query("CREATE TABLE test (test_field TEXT)"); err != nil { suite.T().Error(err) } + + suite.ctx = formatting.ToContext( + context.Background(), + formatting.New().WithTemplate("{{.}}"), + ) + } // Delete Table after test @@ -61,6 +70,27 @@ func (suite *PostgresSetupTestSuite) TestPostgresNewStorage() { "dataField": "test_field", }) assert.NoError(suite.T(), err) + + _, err = NewStorage(map[string]interface{}{ + "databaseUrl": suite.databaseUrl, + "tableName": "test", + "useFormattingToPerformQuery": true, + }) + assert.Error(suite.T(), err) + + _, err = NewStorage(map[string]interface{}{ + "databaseUrl": suite.databaseUrl, + "useFormattingToPerformQuery": true, + "query": "", + }) + assert.Error(suite.T(), err) + + _, err = NewStorage(map[string]interface{}{ + "databaseUrl": suite.databaseUrl, + "useFormattingToPerformQuery": true, + "query": "INSERT INTO test (test_field) VALUES ('$field')", + }) + assert.NoError(suite.T(), err) } func (suite *PostgresSetupTestSuite) TestPostgresPush() { @@ -69,7 +99,7 @@ func (suite *PostgresSetupTestSuite) TestPostgresPush() { "tableName": "Not Exist", "dataField": "Not exist", }) - err := newClient.Push("Hello") + err := newClient.Push(suite.ctx, []byte("Hello")) assert.Error(suite.T(), err) newClient, err = NewStorage(map[string]interface{}{ @@ -79,10 +109,39 @@ func (suite *PostgresSetupTestSuite) TestPostgresPush() { }) assert.NoError(suite.T(), err) - err = newClient.Push("Hello") + err = newClient.Push(suite.ctx, []byte("Hello")) assert.NoError(suite.T(), err) } +func (suite *PostgresSetupTestSuite) TestPostgresPushNewFormattedQuery() { + newClient, err := NewStorage(map[string]interface{}{ + "databaseUrl": suite.databaseUrl, + "useFormattingToPerformQuery": true, + "query": "INSERT INTO test (test_field) VALUES (:field)", + "args": map[string]string{ + "field": "{{.Payload}}", + }, + }) + assert.NoError(suite.T(), err) + + fakePayload := []byte("A strange payload") + err = newClient.Push( + suite.ctx, + fakePayload, + ) + assert.NoError(suite.T(), err) + + rows, err := suite.client.Query("SELECT test_field FROM test") + assert.NoError(suite.T(), err) + + var result string + for rows.Next() { + err := rows.Scan(&result) + assert.NoError(suite.T(), err) + } + assert.Equal(suite.T(), string(fakePayload), result) +} + func TestRunPostgresPush(t *testing.T) { if testing.Short() { t.Skip("postgresql testing is skiped in short version of test") diff --git a/pkg/storage/rabbitmq/rabbitmq.go b/pkg/storage/rabbitmq/rabbitmq.go index a51610d..0d63b99 100644 --- a/pkg/storage/rabbitmq/rabbitmq.go +++ b/pkg/storage/rabbitmq/rabbitmq.go @@ -1,8 +1,8 @@ package rabbitmq import ( + "context" "errors" - "fmt" "time" "github.com/rs/zerolog/log" @@ -105,7 +105,7 @@ func (c *storage) Name() string { // A run is made from external caller // @param value that will be pushed // @return an error if the push failed -func (c *storage) Push(value interface{}) error { +func (c *storage) Push(ctx context.Context, value []byte) error { for attempt := 0; attempt < maxAttempt; attempt++ { err := c.channel.Publish( c.config.Exchange, @@ -114,7 +114,7 @@ func (c *storage) Push(value interface{}) error { c.config.Immediate, amqp.Publishing{ ContentType: c.config.ContentType(), - Body: []byte(fmt.Sprintf("%v", value)), + Body: value, }) if err != nil { diff --git a/pkg/storage/rabbitmq/rabbitmq_test.go b/pkg/storage/rabbitmq/rabbitmq_test.go index 849b54f..3538956 100644 --- a/pkg/storage/rabbitmq/rabbitmq_test.go +++ b/pkg/storage/rabbitmq/rabbitmq_test.go @@ -1,6 +1,7 @@ package rabbitmq import ( + "context" "fmt" "os" "testing" @@ -68,7 +69,7 @@ func (suite *RabbitMQSetupTestSuite) TestRabbitMQPush() { }) assert.NoError(suite.T(), err) - err = newClient.Push("Hello") + err = newClient.Push(context.Background(), []byte("Hello")) assert.NoError(suite.T(), err) } @@ -106,9 +107,9 @@ func (suite *RabbitMQSetupTestSuite) TestReconnect() { }) assert.NoError(suite.T(), err) - assert.NoError(suite.T(), newClient.Push("Hello")) + assert.NoError(suite.T(), newClient.Push(context.Background(), []byte("Hello"))) assert.NoError(suite.T(), newClient.client.Close()) - assert.NoError(suite.T(), newClient.Push("Hello")) + assert.NoError(suite.T(), newClient.Push(context.Background(), []byte("Hello"))) assert.NoError(suite.T(), newClient.channel.Close()) - assert.NoError(suite.T(), newClient.Push("Hello")) + assert.NoError(suite.T(), newClient.Push(context.Background(), []byte("Hello"))) } diff --git a/pkg/storage/redis/redis.go b/pkg/storage/redis/redis.go index b71ef74..836caaa 100644 --- a/pkg/storage/redis/redis.go +++ b/pkg/storage/redis/redis.go @@ -12,7 +12,6 @@ import ( type storage struct { client *redis.Client config *config - ctx context.Context } type config struct { @@ -33,7 +32,6 @@ func NewStorage(configRaw map[string]interface{}) (*storage, error) { newClient := storage{ config: &config{}, - ctx: context.Background(), } if err := valuable.Decode(configRaw, &newClient.config); err != nil { @@ -50,7 +48,7 @@ func NewStorage(configRaw map[string]interface{}) (*storage, error) { ) // Ping Redis for testing config - if err := newClient.client.Ping(newClient.ctx).Err(); err != nil { + if err := newClient.client.Ping(context.Background()).Err(); err != nil { return nil, err } @@ -67,8 +65,8 @@ func (c storage) Name() string { // A run is made from external caller // @param value that will be pushed // @return an error if the push failed -func (c storage) Push(value interface{}) error { - if err := c.client.RPush(c.ctx, c.config.Key, value).Err(); err != nil { +func (c storage) Push(ctx context.Context, value []byte) error { + if err := c.client.RPush(ctx, c.config.Key, value).Err(); err != nil { return err } diff --git a/pkg/storage/redis/redis_test.go b/pkg/storage/redis/redis_test.go index 76498ca..0420049 100644 --- a/pkg/storage/redis/redis_test.go +++ b/pkg/storage/redis/redis_test.go @@ -1,6 +1,7 @@ package redis import ( + "context" "os" "testing" @@ -44,10 +45,7 @@ func (suite *RedisSetupTestSuite) TestRedisPush() { }) assert.NoError(suite.T(), err) - err = newClient.Push(func() {}) - assert.Error(suite.T(), err) - - err = newClient.Push("Hello") + err = newClient.Push(context.Background(), []byte("Hello")) assert.NoError(suite.T(), err) } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index a3e7c97..d446a4e 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -1,6 +1,7 @@ package storage import ( + "context" "fmt" "atomys.codes/webhooked/pkg/storage/postgres" @@ -16,7 +17,7 @@ type Pusher interface { // Will be unique across all storages Name() string // Method call when insert new data in the storage - Push(value interface{}) error + Push(ctx context.Context, value []byte) error } // Load will fetch and return the built-in storage based on the given diff --git a/tests/webhooks.tests.yml b/tests/webhooks.tests.yml index 5aa523c..41f6481 100644 --- a/tests/webhooks.tests.yml +++ b/tests/webhooks.tests.yml @@ -28,4 +28,20 @@ specs: }, "payload": {{ .Payload }} } - storage: [] \ No newline at end of file + storage: + - type: postgres + specs: + databaseUrl: 'postgresql://postgres:postgres@postgres:5432/postgres' + useFormattingToPerformQuery: true + query: | + INSERT INTO webhooks (payload, config, storage, metadata) VALUES (:payload, :config, :storage, :metadata) + args: + payload: '{{ .Payload }}' + config: '{{ toJson .Config }}' + storage: '{{ toJson .Storage }}' + metadata: | + { + "model": "{{ .Request.Header | getHeader "X-Model" }}", + "event": "{{ .Request.Header | getHeader "X-Event" }}", + "deliveryID": "{{ .Request.Header | getHeader "X-Delivery" | default "unknown" }}" + } \ No newline at end of file