Skip to content

Commit

Permalink
Merge pull request #8 from isd-sgcu/feat/s3-sdk
Browse files Browse the repository at this point in the history
feat: migrate to s3 compatible api
  • Loading branch information
bookpanda authored Jul 16, 2024
2 parents f494b06 + f0e134c commit 89e30a3
Show file tree
Hide file tree
Showing 14 changed files with 199 additions and 82 deletions.
13 changes: 13 additions & 0 deletions .devcontainer/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
FROM mcr.microsoft.com/devcontainers/go:1-1.21-bookworm

# [Optional] Uncomment this section to install additional OS packages.
# RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
# && apt-get -y install --no-install-recommends <your-package-list-here>

# [Optional] Uncomment the next lines to use go get to install anything else you need
# USER vscode
# RUN go get -x <your-dependency-or-tool>
# USER root

# [Optional] Uncomment this line to install global node packages.
# RUN su vscode -c "source /usr/local/share/nvm/nvm.sh && npm install -g <your-package-here>" 2>&1
32 changes: 32 additions & 0 deletions .devcontainer/devcontainer.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
// README at: https://github.com/devcontainers/templates/tree/main/src/go-postgres
{
"name": "RPKM67 Auth Service",
"dockerComposeFile": "docker-compose.yml",
"service": "app",
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
"customizations": {
"vscode": {
"extensions": [
"aldijav.golangwithdidi",
"VisualStudioExptTeam.vscodeintellicode",
"VisualStudioExptTeam.vscodeintellicode-completions",
"ZainChen.json",
"ms-kubernetes-tools.vscode-kubernetes-tools",
"ms-vscode.makefile-tools",
"DavidAnson.vscode-markdownlint",
"esbenp.prettier-vscode",
"christian-kohler.path-intellisense",
"zxh404.vscode-proto3",
"redhat.vscode-yaml",
"ms-azuretools.vscode-docker",
"aaron-bond.better-comments"
]
}
},
"forwardPorts": [
5432,
6379
],
"postStartCommand": "make setup"
}
34 changes: 34 additions & 0 deletions .devcontainer/docker-compose.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
version: '3.8'

volumes:
postgres-data:

services:
app:
build:
context: .
dockerfile: Dockerfile
env_file:
- .env

volumes:
- ../..:/workspaces:cached
command: sleep infinity
network_mode: service:db

db:
image: postgres:latest
restart: unless-stopped
network_mode: service:redis
volumes:
- ./postgres-data:/var/lib/postgresql/data
env_file:
- .env

redis:
image: redis:latest
restart: unless-stopped
env_file:
- .env
volumes:
- ./redis-data:/data
12 changes: 12 additions & 0 deletions .github/dependabot.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
# To get started with Dependabot version updates, you'll need to specify which
# package ecosystems to update and where the package manifests are located.
# Please see the documentation for more information:
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
# https://containers.dev/guide/dependabot

version: 2
updates:
- package-ecosystem: "devcontainers"
directory: "/"
schedule:
interval: weekly
4 changes: 3 additions & 1 deletion .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -32,4 +32,6 @@ coverage.html

tmp
staff.json
docker-compose.qa.yml
docker-compose.qa.yml

.devcontainer/*-data
24 changes: 15 additions & 9 deletions cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,12 @@ import (
"github.com/isd-sgcu/rpkm67-store/internal/object"
"github.com/isd-sgcu/rpkm67-store/internal/utils"
"github.com/isd-sgcu/rpkm67-store/logger"
"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"

"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/aws/credentials"
"github.com/aws/aws-sdk-go/aws/session"
"github.com/aws/aws-sdk-go/service/s3"

"go.uber.org/zap"
"google.golang.org/grpc"
"google.golang.org/grpc/health"
Expand All @@ -34,15 +38,17 @@ func main() {

logger := logger.New(conf)

minioClient, err := minio.New(conf.Store.Endpoint, &minio.Options{
Creds: credentials.NewStaticV4(conf.Store.AccessKey, conf.Store.SecretKey, ""),
Secure: conf.Store.UseSSL,
})
if err != nil {
panic(fmt.Sprintf("Failed to connect to Minio: %v", err))
s3Config := &aws.Config{
Credentials: credentials.NewStaticCredentials(conf.Store.AccessKey, conf.Store.SecretKey, ""),
Endpoint: aws.String(conf.Store.Endpoint),
Region: aws.String(conf.Store.Region),
S3ForcePathStyle: aws.Bool(false),
}

storeClient := store.NewClient(minioClient)
s3Session := session.Must(session.NewSession(s3Config))
s3Client := s3.New(s3Session)

storeClient := store.NewClient(s3Client)
httpClient := &http.Client{}

randomUtils := utils.NewRandomUtils()
Expand Down
27 changes: 15 additions & 12 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,14 @@ type DB struct {
}

type Store struct {
Endpoint string
AccessKey string
SecretKey string
UseSSL bool
BucketName string
Region string
Token string
Endpoint string
CDNEndpoint string
AccessKey string
SecretKey string
UseSSL bool
BucketName string
Region string
Token string
}

type Config struct {
Expand All @@ -44,11 +45,13 @@ func LoadConfig() (config *Config, err error) {
}

storeConfig := Store{
BucketName: os.Getenv("STORE_BUCKET_NAME"),
Endpoint: os.Getenv("STORE_ENDPOINT"),
AccessKey: os.Getenv("STORE_ACCESS_KEY"),
SecretKey: os.Getenv("STORE_SECRET_KEY"),
UseSSL: os.Getenv("STORE_USE_SSL") == "true",
BucketName: os.Getenv("STORE_BUCKET_NAME"),
Endpoint: os.Getenv("STORE_ENDPOINT"),
CDNEndpoint: os.Getenv("STORE_CDN_ENDPOINT"),
AccessKey: os.Getenv("STORE_ACCESS_KEY"),
SecretKey: os.Getenv("STORE_SECRET_KEY"),
Region: os.Getenv("STORE_REGION"),
UseSSL: os.Getenv("STORE_USE_SSL") == "true",
}

return &Config{
Expand Down
2 changes: 2 additions & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -9,10 +9,12 @@ require (
)

require (
github.com/aws/aws-sdk-go v1.54.19 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/google/uuid v1.6.0 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/compress v1.17.6 // indirect
github.com/klauspost/cpuid/v2 v2.2.7 // indirect
github.com/minio/md5-simd v1.1.2 // indirect
Expand Down
8 changes: 8 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
github.com/aws/aws-sdk-go v1.54.19 h1:tyWV+07jagrNiCcGRzRhdtVjQs7Vy41NwsuOcl0IbVI=
github.com/aws/aws-sdk-go v1.54.19/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
Expand All @@ -16,6 +19,9 @@ github.com/isd-sgcu/rpkm67-go-proto v0.1.7 h1:FO8B4A7iuVS8PM4cEXe3HxZxlqdDY8va1z
github.com/isd-sgcu/rpkm67-go-proto v0.1.7/go.mod h1:Z5SYz5kEe4W+MdqPouF0zEOiaqvg+s9I1S5d0q6e+Jw=
github.com/isd-sgcu/rpkm67-go-proto v0.2.0 h1:tPfNgCuqS4g0f+2hzcpY+8hYXSa7DZDPvRejRzOk2cI=
github.com/isd-sgcu/rpkm67-go-proto v0.2.0/go.mod h1:Z5SYz5kEe4W+MdqPouF0zEOiaqvg+s9I1S5d0q6e+Jw=
github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/klauspost/compress v1.17.6 h1:60eq2E/jlfwQXtvZEeBUYADs+BwKBWURIY+Gj2eRGjI=
Expand All @@ -33,6 +39,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rs/xid v1.5.0 h1:mKX4bl4iPYJtEIxp6CYiUuLQ/8DYMoz0PUdtGgMFRVc=
github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
Expand Down Expand Up @@ -83,5 +90,6 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
32 changes: 21 additions & 11 deletions internal/client/store/store.client.go
Original file line number Diff line number Diff line change
@@ -1,29 +1,39 @@
package store

import (
"bytes"
"context"
"io"

"github.com/minio/minio-go/v7"
"github.com/aws/aws-sdk-go/aws"
"github.com/aws/aws-sdk-go/service/s3"
)

type Client interface {
PutObject(ctx context.Context, bucketName string, objectName string, reader io.Reader, objectSize int64, opts minio.PutObjectOptions) (info minio.UploadInfo, err error)
RemoveObject(ctx context.Context, bucketName string, objectName string, opts minio.RemoveObjectOptions) error
PutObject(ctx context.Context, bucketName string, objectName string, reader *bytes.Reader) (info *s3.PutObjectOutput, err error)
RemoveObject(ctx context.Context, bucketName string, objectName string) error
}

type clientImpl struct {
*minio.Client
*s3.S3
}

func NewClient(minioClient *minio.Client) Client {
return &clientImpl{minioClient}
func NewClient(s3Client *s3.S3) Client {
return &clientImpl{s3Client}
}

func (c *clientImpl) PutObject(ctx context.Context, bucketName string, objectName string, reader io.Reader, objectSize int64, opts minio.PutObjectOptions) (info minio.UploadInfo, err error) {
return c.Client.PutObject(ctx, bucketName, objectName, reader, objectSize, opts)
func (c *clientImpl) PutObject(ctx context.Context, bucketName string, objectName string, reader *bytes.Reader) (info *s3.PutObjectOutput, err error) {
return c.S3.PutObject(&s3.PutObjectInput{
Bucket: &bucketName,
Key: &objectName,
Body: reader,
ACL: aws.String(s3.ObjectCannedACLPublicRead),
})
}

func (c *clientImpl) RemoveObject(ctx context.Context, bucketName string, objectName string, opts minio.RemoveObjectOptions) error {
return c.Client.RemoveObject(ctx, bucketName, objectName, opts)
func (c *clientImpl) RemoveObject(ctx context.Context, bucketName string, objectName string) error {
_, err := c.S3.DeleteObject(&s3.DeleteObjectInput{
Bucket: &bucketName,
Key: &objectName,
})
return err
}
13 changes: 4 additions & 9 deletions internal/object/object.repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"github.com/isd-sgcu/rpkm67-store/config"
httpClient "github.com/isd-sgcu/rpkm67-store/internal/client/http"
storeClient "github.com/isd-sgcu/rpkm67-store/internal/client/store"
"github.com/minio/minio-go/v7"
"github.com/pkg/errors"
)

Expand Down Expand Up @@ -42,24 +41,20 @@ func (r *repositoryImpl) Upload(file []byte, bucketName string, objectKey string

buffer := bytes.NewReader(file)

uploadOutput, err := r.storeClient.PutObject(ctx, bucketName, objectKey, buffer,
buffer.Size(), minio.PutObjectOptions{})
_, err = r.storeClient.PutObject(ctx, bucketName, objectKey, buffer)
if err != nil {
return "", "", errors.Wrap(err, fmt.Sprintf("Couldn't upload object to %v/%v.", bucketName, objectKey))
}

return r.GetURL(bucketName, objectKey), uploadOutput.Key, nil
return r.GetURL(bucketName, objectKey), objectKey, nil
}

func (r *repositoryImpl) Delete(bucketName string, objectKey string) (err error) {
ctx := context.Background()
_, cancel := context.WithTimeout(ctx, 50*time.Second)
defer cancel()

opts := minio.RemoveObjectOptions{
GovernanceBypass: true,
}
err = r.storeClient.RemoveObject(ctx, bucketName, objectKey, opts)
err = r.storeClient.RemoveObject(ctx, bucketName, objectKey)
if err != nil {
return errors.Wrap(err, fmt.Sprintf("Couldn't delete object %v/%v.", bucketName, objectKey))
}
Expand All @@ -86,5 +81,5 @@ func (r *repositoryImpl) Get(bucketName string, objectKey string) (url string, e
}

func (r *repositoryImpl) GetURL(bucketName string, objectKey string) string {
return "https://" + r.conf.Endpoint + "/" + bucketName + "/" + objectKey
return r.conf.CDNEndpoint + "/" + bucketName + "/" + objectKey
}
Loading

0 comments on commit 89e30a3

Please sign in to comment.