diff --git a/README.md b/README.md index 9e40ab7..98686ee 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # pg-subsetter -[![lint](https://github.com/teamniteo/pg-subsetter/actions/workflows/lint.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/lint.yml) [![build](https://github.com/teamniteo/pg-subsetter/actions/workflows/go.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/go.yml) [![vuln](https://github.com/teamniteo/pg-subsetter/actions/workflows/vuln.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/vuln.yml) +[![lint](https://github.com/teamniteo/pg-subsetter/actions/workflows/lint.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/lint.yml) [![build](https://github.com/teamniteo/pg-subsetter/actions/workflows/go.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/go.yml) [![vuln](https://github.com/teamniteo/pg-subsetter/actions/workflows/vuln.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/vuln.yml) [![release](https://github.com/teamniteo/pg-subsetter/actions/workflows/release.yml/badge.svg)](https://github.com/teamniteo/pg-subsetter/actions/workflows/release.yml) `pg-subsetter` is a tool designed to synchronize a fraction of a PostgreSQL database to another PostgreSQL database on the fly, it does not copy the SCHEMA. @@ -47,7 +47,7 @@ Usage of subsetter: Prepare schema in target database: ```bash -pg_dump --schema-only -n public -f schemadump.sql "postgres://test_source@localhost:5432/test_source?sslmode=disable" +pg_dump --schema-only --no-owner --no-acl -n public -f schemadump.sql "postgres://test_source@localhost:5432/test_source?sslmode=disable" psql -f schemadump.sql "postgres://test_target@localhost:5432/test_target?sslmode=disable" ``` diff --git a/subsetter/sync.go b/subsetter/sync.go index 570c4f1..2130e13 100644 --- a/subsetter/sync.go +++ b/subsetter/sync.go @@ -2,6 +2,7 @@ package subsetter import ( "context" + "fmt" "sort" "github.com/jackc/pgx/v5/pgconn" @@ -25,6 +26,13 @@ type Rule struct { Where string } +func (r Rule) Query() string { + if r.Where == "" { + return fmt.Sprintf("SELECT * FROM %s", r.Table) + } + return fmt.Sprintf("SELECT * FROM %s WHERE %s", r.Table, r.Where) +} + func NewSync(source string, target string, fraction float64, include []Rule, exclude []Rule, verbose bool) (*Sync, error) { src, err := pgxpool.New(context.Background(), source) if err != nil { @@ -119,7 +127,7 @@ func (s *Sync) CopyTables(tables []Table) (err error) { if include.Table == table.Name { log.Info().Str("query", include.Where).Msgf("Selecting forced rows for table %s", table.Name) var data string - if data, err = CopyQueryToString(include.Where, s.source); err != nil { + if data, err = CopyQueryToString(include.Query(), s.source); err != nil { return errors.Wrapf(err, "Error copying forced rows for table %s", table.Name) } if err = CopyStringToTable(table.Name, data, s.destination); err != nil { @@ -152,7 +160,7 @@ func (s *Sync) CopyTables(tables []Table) (err error) { } if err = CopyStringToTable(relation.PrimaryTable, data, s.destination); err != nil { if condition, ok := err.(*pgconn.PgError); ok && condition.Code == "23503" { - log.Warn().Msgf("Skipping %s because of foreign key violation", relation.PrimaryTable) + log.Warn().Msgf("Skipping %s because of cyclic foreign key", relation.PrimaryTable) err = nil } else { return errors.Wrapf(err, "Error inserting related table %s", relation.PrimaryTable)