diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index 5934261..8b26191 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -10,9 +10,10 @@ on: branches: [ "main" ] jobs: + vuln: + uses: cristalhq/.github/.github/workflows/vuln.yml@v0.4.0 build: - runs-on: ubuntu-latest services: postgres: @@ -37,10 +38,16 @@ jobs: - name: Set up Go uses: actions/setup-go@v4 with: - go-version: '1.20' + go-version: stable - name: Build - run: go build -v ./... + run: make build - name: Test run: go test -v ./... + + - name: Upload assets + uses: actions/upload-artifact@v3 + with: + name: pg-subsetter + path: dist/* diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..27aad41 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,38 @@ +name: release + +permissions: + contents: write + +on: + workflow_call: + inputs: + tag: + description: 'Tag to create' + required: true + default: 'v0.0.0' + type: string + +jobs: + run: + name: Release + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + + - name: Checkout with tags + run: git fetch --prune --unshallow --tags + + - name: Create release + run: | + git log --format="%C(auto) %H %s" `git tag --sort=-committerdate | head -1`...HEAD > changelog.txt + echo ${{ secrets.GITHUB_TOKEN }} | gh auth login --with-token + gh release create ${{ github.event.inputs.tag }} -t ${{ github.event.inputs.tag }} -F changelog.txt + + - name: Run GoReleaser + uses: goreleaser/goreleaser-action@v4.4.0 + with: + version: latest + args: release --rm-dist + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 5555cbc..6f0de07 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ -.pgsql/data \ No newline at end of file +.pgsql/data +bin +dist/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000..e7b8aa6 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,36 @@ +before: + hooks: + - go mod tidy +builds: + - env: + - CGO_ENABLED=0 + goos: + - linux + - windows + - darwin + main: ./cli + +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of uname. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} + # use zip for windows archives + format_overrides: + - goos: windows + format: zip +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - '^docs:' + - '^test:' \ No newline at end of file diff --git a/Makefile b/Makefile index 7c68b83..6728bfb 100644 --- a/Makefile +++ b/Makefile @@ -15,3 +15,7 @@ is-postgres-running: .PHONY: pgweb pgweb:is-postgres-running @pgweb --url "postgres://test_source@localhost:5432/test_source?sslmode=disable" + +build: + rm -rf dist + goreleaser build --snapshot --clean diff --git a/README.md b/README.md index 5f17498..2e537bc 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,19 @@ Utilizing the native PostgreSQL COPY command, pg-subsetter performs data transfe ## Usage +``` +Usage of subsetter: + -dst string + Destination DSN + -f float + Fraction of rows to copy (default 0.05) + -src string + Source DSN +``` + + +Example: + ```pg-subsetter -src postgresql://:@/bigdb -dst postgresql://:@/littledb -f 0.05``` # Installing diff --git a/cli/main.go b/cli/main.go index 55a25a6..dd07c59 100644 --- a/cli/main.go +++ b/cli/main.go @@ -6,12 +6,20 @@ import ( "github.com/rs/zerolog/log" ) -var src = flag.String("src", "", "Source DSN") -var dst = flag.String("dst", "", "Destination DSN") -var fraction = flag.Float64("f", 1.0, "Fraction of rows to copy") +var src = flag.String("src", "", "Source database DSN") +var dst = flag.String("dst", "", "Destination database DSN") +var fraction = flag.Float64("f", 0.05, "Fraction of rows to copy") func main() { flag.Parse() log.Info().Msg("Starting") + if *src == "" || *dst == "" { + log.Fatal().Msg("Source and destination DSNs are required") + } + + if *fraction <= 0 || *fraction > 1 { + log.Fatal().Msg("Fraction must be between 0 and 1") + } + } diff --git a/flake.nix b/flake.nix index fbc3669..e355e0d 100644 --- a/flake.nix +++ b/flake.nix @@ -33,6 +33,7 @@ stdenv = stdenvMinimal; packages = with pkgs; [ go + goreleaser postgresql process-compose shellcheck diff --git a/go.mod b/go.mod index c8f7e99..f20e593 100644 --- a/go.mod +++ b/go.mod @@ -10,11 +10,9 @@ require ( ) require ( - github.com/jackc/pgx v3.6.2+incompatible // indirect github.com/jackc/pgx/v5 v5.4.3 github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-isatty v0.0.14 // indirect - github.com/pkg/errors v0.9.1 // indirect golang.org/x/crypto v0.12.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/text v0.12.0 // indirect diff --git a/go.sum b/go.sum index 6d2fa97..ffd980e 100644 --- a/go.sum +++ b/go.sum @@ -1,20 +1,19 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= -github.com/jackc/pgx v3.6.2+incompatible h1:2zP5OD7kiyR3xzRYMhOcXVvkDZsImVXfj+yIyTQf3/o= -github.com/jackc/pgx v3.6.2+incompatible/go.mod h1:0ZGrqGqkRlliWnWB4zKnWtjbSWbGkVEFm4TeybAXq+I= github.com/jackc/pgx/v5 v5.4.3 h1:cxFyXhxlvAifxnkKKdlxv8XqUf59tDlYjnV5YYfsJJY= github.com/jackc/pgx/v5 v5.4.3/go.mod h1:Ig06C2Vu0t5qXC60W8sqIthScaEnFvojjj9dSljmHRA= github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= @@ -22,6 +21,7 @@ github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiS github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -32,3 +32,4 @@ golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/subsetter/db_test.go b/subsetter/db_test.go index 543599a..cc17233 100644 --- a/subsetter/db_test.go +++ b/subsetter/db_test.go @@ -41,8 +41,7 @@ func populateTestsWithData(conn *pgx.Conn, table string, size int) { for i := 0; i < size; i++ { query := fmt.Sprintf("INSERT INTO %s (text) VALUES ('test%d') RETURNING id", table, i) var row string - err := conn.QueryRow(context.Background(), query).Scan(&row) - fmt.Println(err) + conn.QueryRow(context.Background(), query).Scan(&row) query = fmt.Sprintf("INSERT INTO relation (simple_id) VALUES ('%v')", row) conn.Exec(context.Background(), query)