From 9f8a3b64a92cb15c294da201dfad85941e76d667 Mon Sep 17 00:00:00 2001 From: Edouard Paris Date: Wed, 4 Jul 2018 15:53:00 +0200 Subject: [PATCH 1/6] refac application: fileFromKVStore and processImage --- application/application.go | 132 +++++++++++++++++++------------------ 1 file changed, 68 insertions(+), 64 deletions(-) diff --git a/application/application.go b/application/application.go index dc0fe573..55cf17d7 100644 --- a/application/application.go +++ b/application/application.go @@ -193,26 +193,11 @@ func Delete(ctx context.Context, filepath string) error { // ImageFileFromContext generates an ImageFile from gin context func ImageFileFromContext(c *gin.Context, async bool, load bool) (*image.ImageFile, error) { - key := c.MustGet("key").(string) + storeKey := c.MustGet("key").(string) k := kvstore.FromContext(c) - - cfg := config.FromContext(c) - l := logger.FromContext(c) - destStorage := storage.DestinationFromContext(c) - - var file = &image.ImageFile{ - Key: key, - Storage: destStorage, - Headers: map[string]string{}, - } - var err error - var filepath string - - storeKey := key - // Image from the KVStore found imageKey, err := k.Get(storeKey) if err != nil { @@ -220,77 +205,96 @@ func ImageFileFromContext(c *gin.Context, async bool, load bool) (*image.ImageFi } if imageKey != nil { - stored, err := conv.String(imageKey) - if err != nil { - return nil, err - } - - file.Filepath = stored - - l.Infof("Key %s found in kvstore: %s", storeKey, stored) - - if load { - file, err = image.FromStorage(destStorage, stored) - - if err != nil { - return nil, err - } - } + return fileFromStorage(c, l, storeKey, imageKey, load) } else { - l.Infof("Key %s not found in kvstore", storeKey) - - u, exists := c.Get("url") - - parameters := c.MustGet("parameters").(map[string]string) - // Image not found from the KVStore, we need to process it // URL available in Query String - if exists { - file, err = image.FromURL(u.(*url.URL), cfg.Options.DefaultUserAgent) - } else { - // URL provided we use http protocol to retrieve it - s := storage.SourceFromContext(c) + l.Infof("Key %s not found in kvstore", storeKey) + return processImage(c, l, storeKey, async) + } +} - filepath = parameters["path"] +func fileFromStorage(c *gin.Context, l logger.Logger, storeKey string, imageKey interface{}, load bool) (*image.ImageFile, error) { + stored, err := conv.String(imageKey) + if err != nil { + return nil, err + } - if !s.Exists(filepath) { - return nil, errs.ErrFileNotExists - } + l.Infof("Key %s found in kvstore: %s", storeKey, stored) - file, err = image.FromStorage(s, filepath) - } + destStorage := storage.DestinationFromContext(c) + + file := &image.ImageFile{ + Key: storeKey, + Storage: destStorage, + Filepath: stored, + Headers: map[string]string{}, + } + if load { + file, err = image.FromStorage(destStorage, stored) if err != nil { return nil, err } + } - op := c.MustGet("op").(engine.Operation) + file.Headers["ETag"] = storeKey + return file, nil +} - file, err = engine.FromContext(c).Transform(file, op, parameters) +func processImage(c *gin.Context, l logger.Logger, storeKey string, async bool) (*image.ImageFile, error) { + cfg := config.FromContext(c) + destStorage := storage.DestinationFromContext(c) - if err != nil { - return nil, err + file := &image.ImageFile{ + Key: storeKey, + Storage: destStorage, + Headers: map[string]string{}, + } + + var filepath string + parameters := c.MustGet("parameters").(map[string]string) + + var err error + u, exists := c.Get("url") + if exists { + file, err = image.FromURL(u.(*url.URL), cfg.Options.DefaultUserAgent) + } else { + // URL provided we use http protocol to retrieve it + s := storage.SourceFromContext(c) + + filepath = parameters["path"] + if !s.Exists(filepath) { + return nil, errs.ErrFileNotExists } - filename := ShardFilename(c, key) + file, err = image.FromStorage(s, filepath) + } + if err != nil { + return nil, err + } - file.Filepath = fmt.Sprintf("%s.%s", filename, file.Format()) + op := c.MustGet("op").(engine.Operation) + file, err = engine.FromContext(c).Transform(file, op, parameters) + if err != nil { + return nil, err } - file.Key = key + filename := ShardFilename(c, storeKey) + file.Filepath = fmt.Sprintf("%s.%s", filename, file.Format()) file.Storage = destStorage + file.Headers["ETag"] = storeKey - file.Headers["ETag"] = key - - if imageKey == nil { - if async == true { - go Store(c, filepath, file) - } else { - err = Store(c, filepath, file) + if async == true { + go Store(c, filepath, file) + } else { + err = Store(c, filepath, file) + if err != nil { + return nil, err } } - return file, err + return file, nil } // ShardFilename shards a filename based on config From 3a2b280b576bb574f419ff887f45eb32cd1ae071 Mon Sep 17 00:00:00 2001 From: Edouard Paris Date: Wed, 4 Jul 2018 16:50:05 +0200 Subject: [PATCH 2/6] fix application: keep storeKey in file --- application/application.go | 1 + 1 file changed, 1 insertion(+) diff --git a/application/application.go b/application/application.go index 55cf17d7..d034d45d 100644 --- a/application/application.go +++ b/application/application.go @@ -283,6 +283,7 @@ func processImage(c *gin.Context, l logger.Logger, storeKey string, async bool) filename := ShardFilename(c, storeKey) file.Filepath = fmt.Sprintf("%s.%s", filename, file.Format()) file.Storage = destStorage + file.Key = storeKey file.Headers["ETag"] = storeKey if async == true { From 29cfbca159a2ddd762c86ca2920b37cf99e16e77 Mon Sep 17 00:00:00 2001 From: Edouard Paris Date: Thu, 5 Jul 2018 14:06:44 +0200 Subject: [PATCH 3/6] fix application ImageFileFromContext --- application/application.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/application/application.go b/application/application.go index d034d45d..392ec719 100644 --- a/application/application.go +++ b/application/application.go @@ -206,12 +206,12 @@ func ImageFileFromContext(c *gin.Context, async bool, load bool) (*image.ImageFi if imageKey != nil { return fileFromStorage(c, l, storeKey, imageKey, load) - } else { - // Image not found from the KVStore, we need to process it - // URL available in Query String - l.Infof("Key %s not found in kvstore", storeKey) - return processImage(c, l, storeKey, async) } + + // Image not found from the KVStore, we need to process it + // URL available in Query String + l.Infof("Key %s not found in kvstore", storeKey) + return processImage(c, l, storeKey, async) } func fileFromStorage(c *gin.Context, l logger.Logger, storeKey string, imageKey interface{}, load bool) (*image.ImageFile, error) { From 153b05fcff219dae3740f958d70ebc2a3622442a Mon Sep 17 00:00:00 2001 From: Edouard Paris Date: Thu, 5 Jul 2018 15:47:31 +0200 Subject: [PATCH 4/6] fix util: remove SortMapString --- middleware/parsers.go | 7 ++----- signature/signature.go | 19 ++----------------- util/util.go | 21 --------------------- 3 files changed, 4 insertions(+), 43 deletions(-) diff --git a/middleware/parsers.go b/middleware/parsers.go index b5c19233..9566c6c9 100644 --- a/middleware/parsers.go +++ b/middleware/parsers.go @@ -11,7 +11,6 @@ import ( "github.com/thoas/picfit/engine" "github.com/thoas/picfit/hash" "github.com/thoas/picfit/image" - "github.com/thoas/picfit/util" ) const sigParamName = "sig" @@ -64,11 +63,9 @@ func KeyParser() gin.HandlerFunc { queryString[k] = v[0] } - sorted := util.SortMapString(queryString) + delete(queryString, sigParamName) - delete(sorted, sigParamName) - - serialized := hash.Serialize(sorted) + serialized := hash.Serialize(queryString) key := hash.Tokey(serialized) diff --git a/signature/signature.go b/signature/signature.go index 1b2c2791..4dc91fa6 100644 --- a/signature/signature.go +++ b/signature/signature.go @@ -7,8 +7,6 @@ import ( "fmt" "net/url" "regexp" - - "github.com/thoas/picfit/util" ) var signRegex = regexp.MustCompile("&?sig=[^&]*") @@ -17,7 +15,7 @@ var signRegex = regexp.MustCompile("&?sig=[^&]*") func VerifyParameters(key string, qs map[string]string) bool { params := url.Values{} - for k, v := range util.SortMapString(qs) { + for k, v := range qs { params.Set(k, v) } @@ -37,24 +35,11 @@ func Sign(key string, qs string) string { // SignRaw encodes raw query string (not sorted) using a key func SignRaw(key string, queryString string) (string, error) { values, err := url.ParseQuery(queryString) - if err != nil { return "", err } - qs := url.Values{} - - params := map[string]string{} - - for k, v := range values { - params[k] = v[0] - } - - for k, v := range util.SortMapString(params) { - qs.Set(k, v) - } - - return Sign(key, qs.Encode()), nil + return Sign(key, values.Encode()), nil } // AppendSign appends the signature to query string diff --git a/util/util.go b/util/util.go index b746e670..f54fca57 100644 --- a/util/util.go +++ b/util/util.go @@ -2,7 +2,6 @@ package util import ( "fmt" - "sort" ) func MapInterfaceToMapString(obj map[string]interface{}) map[string]string { @@ -14,23 +13,3 @@ func MapInterfaceToMapString(obj map[string]interface{}) map[string]string { return results } - -func SortMapString(obj map[string]string) map[string]string { - mk := make([]string, len(obj)) - - i := 0 - for k, _ := range obj { - mk[i] = k - i++ - } - - sort.Strings(mk) - - results := make(map[string]string) - - for _, index := range mk { - results[index] = obj[index] - } - - return results -} From 701e376f4457eb9f3d894b23edce14255c02894c Mon Sep 17 00:00:00 2001 From: Edouard Paris Date: Mon, 9 Jul 2018 10:54:54 +0200 Subject: [PATCH 5/6] Revert "fix util: remove SortMapString" This reverts commit 153b05fcff219dae3740f958d70ebc2a3622442a. --- middleware/parsers.go | 7 +++++-- signature/signature.go | 19 +++++++++++++++++-- util/util.go | 21 +++++++++++++++++++++ 3 files changed, 43 insertions(+), 4 deletions(-) diff --git a/middleware/parsers.go b/middleware/parsers.go index 9566c6c9..b5c19233 100644 --- a/middleware/parsers.go +++ b/middleware/parsers.go @@ -11,6 +11,7 @@ import ( "github.com/thoas/picfit/engine" "github.com/thoas/picfit/hash" "github.com/thoas/picfit/image" + "github.com/thoas/picfit/util" ) const sigParamName = "sig" @@ -63,9 +64,11 @@ func KeyParser() gin.HandlerFunc { queryString[k] = v[0] } - delete(queryString, sigParamName) + sorted := util.SortMapString(queryString) - serialized := hash.Serialize(queryString) + delete(sorted, sigParamName) + + serialized := hash.Serialize(sorted) key := hash.Tokey(serialized) diff --git a/signature/signature.go b/signature/signature.go index 4dc91fa6..1b2c2791 100644 --- a/signature/signature.go +++ b/signature/signature.go @@ -7,6 +7,8 @@ import ( "fmt" "net/url" "regexp" + + "github.com/thoas/picfit/util" ) var signRegex = regexp.MustCompile("&?sig=[^&]*") @@ -15,7 +17,7 @@ var signRegex = regexp.MustCompile("&?sig=[^&]*") func VerifyParameters(key string, qs map[string]string) bool { params := url.Values{} - for k, v := range qs { + for k, v := range util.SortMapString(qs) { params.Set(k, v) } @@ -35,11 +37,24 @@ func Sign(key string, qs string) string { // SignRaw encodes raw query string (not sorted) using a key func SignRaw(key string, queryString string) (string, error) { values, err := url.ParseQuery(queryString) + if err != nil { return "", err } - return Sign(key, values.Encode()), nil + qs := url.Values{} + + params := map[string]string{} + + for k, v := range values { + params[k] = v[0] + } + + for k, v := range util.SortMapString(params) { + qs.Set(k, v) + } + + return Sign(key, qs.Encode()), nil } // AppendSign appends the signature to query string diff --git a/util/util.go b/util/util.go index f54fca57..b746e670 100644 --- a/util/util.go +++ b/util/util.go @@ -2,6 +2,7 @@ package util import ( "fmt" + "sort" ) func MapInterfaceToMapString(obj map[string]interface{}) map[string]string { @@ -13,3 +14,23 @@ func MapInterfaceToMapString(obj map[string]interface{}) map[string]string { return results } + +func SortMapString(obj map[string]string) map[string]string { + mk := make([]string, len(obj)) + + i := 0 + for k, _ := range obj { + mk[i] = k + i++ + } + + sort.Strings(mk) + + results := make(map[string]string) + + for _, index := range mk { + results[index] = obj[index] + } + + return results +} From 3530bf29c1ef1dbef96d5179055f8c776ceffe1a Mon Sep 17 00:00:00 2001 From: Edouard Paris Date: Mon, 9 Jul 2018 10:57:06 +0200 Subject: [PATCH 6/6] fix sig: no need to sort values map before encode --- signature/signature.go | 19 ++----------------- 1 file changed, 2 insertions(+), 17 deletions(-) diff --git a/signature/signature.go b/signature/signature.go index 1b2c2791..4dc91fa6 100644 --- a/signature/signature.go +++ b/signature/signature.go @@ -7,8 +7,6 @@ import ( "fmt" "net/url" "regexp" - - "github.com/thoas/picfit/util" ) var signRegex = regexp.MustCompile("&?sig=[^&]*") @@ -17,7 +15,7 @@ var signRegex = regexp.MustCompile("&?sig=[^&]*") func VerifyParameters(key string, qs map[string]string) bool { params := url.Values{} - for k, v := range util.SortMapString(qs) { + for k, v := range qs { params.Set(k, v) } @@ -37,24 +35,11 @@ func Sign(key string, qs string) string { // SignRaw encodes raw query string (not sorted) using a key func SignRaw(key string, queryString string) (string, error) { values, err := url.ParseQuery(queryString) - if err != nil { return "", err } - qs := url.Values{} - - params := map[string]string{} - - for k, v := range values { - params[k] = v[0] - } - - for k, v := range util.SortMapString(params) { - qs.Set(k, v) - } - - return Sign(key, qs.Encode()), nil + return Sign(key, values.Encode()), nil } // AppendSign appends the signature to query string