From 8f3ed77ca44a973647420251a5d5e37b1d7e0e05 Mon Sep 17 00:00:00 2001 From: Stefan Majewsky Date: Thu, 12 Dec 2024 14:23:19 +0100 Subject: [PATCH 1/2] use new go-bits/easypg API --- .gitignore | 2 +- go.mod | 6 +- go.sum | 8 +- internal/api/api_test.go | 4 + internal/collector/shared_test.go | 6 + internal/datamodel/quota_overrides_test.go | 5 + internal/db/connection.go | 35 +- internal/test/setup.go | 55 +-- main.go | 10 +- .../gophercloud/gophercloud/v2/CHANGELOG.md | 8 + .../gophercloud/gophercloud/v2/Makefile | 49 +-- .../openstack/baremetal/v1/nodes/requests.go | 4 + .../openstack/baremetal/v1/nodes/results.go | 86 ++++- .../compute/v2/hypervisors/results.go | 76 ++-- .../gophercloud/v2/provider_client.go | 2 +- .../github.com/sapcc/go-bits/easypg/easypg.go | 28 +- .../sapcc/go-bits/easypg/testhelpers.go | 54 --- .../sapcc/go-bits/easypg/testsetup.go | 332 ++++++++++++++++++ vendor/github.com/sapcc/go-bits/easypg/url.go | 11 +- vendor/modules.txt | 4 +- 20 files changed, 564 insertions(+), 221 deletions(-) create mode 100644 vendor/github.com/sapcc/go-bits/easypg/testsetup.go diff --git a/.gitignore b/.gitignore index 4d1e63345..3cb02c66b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ build/ # test artifacts -testing/postgres* +/.testdb *.actual # custom configuration files used during development diff --git a/go.mod b/go.mod index 728f77b70..55b376ced 100644 --- a/go.mod +++ b/go.mod @@ -7,14 +7,15 @@ require ( github.com/dlmiddlecote/sqlstats v1.0.2 github.com/go-gorp/gorp/v3 v3.1.0 github.com/gofrs/uuid/v5 v5.3.0 - github.com/gophercloud/gophercloud/v2 v2.2.0 + github.com/gophercloud/gophercloud/v2 v2.3.0 github.com/gorilla/mux v1.8.1 + github.com/lib/pq v1.10.9 github.com/majewsky/schwift/v2 v2.0.0 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/common v0.61.0 github.com/rs/cors v1.11.1 github.com/sapcc/go-api-declarations v1.13.1 - github.com/sapcc/go-bits v0.0.0-20241206132118-f18a227dc952 + github.com/sapcc/go-bits v0.0.0-20241212131355-30c23561fbfd go.uber.org/automaxprocs v1.6.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -29,7 +30,6 @@ require ( github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect - github.com/lib/pq v1.10.9 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect diff --git a/go.sum b/go.sum index dae43b1fd..3a851a85d 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/gophercloud/gophercloud/v2 v2.2.0 h1:STqqnSXuhcg1OPBOZ14z6JDm8fKIN13H2bJg6bBuHp8= -github.com/gophercloud/gophercloud/v2 v2.2.0/go.mod h1:f2hMRC7Kakbv5vM7wSGHrIPZh6JZR60GVHryJlF/K44= +github.com/gophercloud/gophercloud/v2 v2.3.0 h1:5ipI2Mgxee0TwQxqnOIUdTbzL4ZBB8GORyZko+yGXI0= +github.com/gophercloud/gophercloud/v2 v2.3.0/go.mod h1:uJWNpTgJPSl2gyzJqcU/pIAhFUWvIkp8eE8M15n9rs4= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -159,8 +159,8 @@ github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/sapcc/go-api-declarations v1.13.1 h1:rovCnLscnoZaIZPWhohSYHzwwYjOnCPsRw3zwtu4tLI= github.com/sapcc/go-api-declarations v1.13.1/go.mod h1:83R3hTANhuRXt/pXDby37IJetw8l7DG41s33Tp9NXxI= -github.com/sapcc/go-bits v0.0.0-20241206132118-f18a227dc952 h1:tg1xF/eh6kM3Ti5AmMCV6zNKRPHtmP8Yh1XYG0/WbSk= -github.com/sapcc/go-bits v0.0.0-20241206132118-f18a227dc952/go.mod h1:ROdTmzQj/gn6dUaxhrCQCTsZtaFJPAFy3CeTD2m/z3k= +github.com/sapcc/go-bits v0.0.0-20241212131355-30c23561fbfd h1:w70x4iw6MjZwRXjFARt3xfDjBq/MunXRdToH+wvFvF8= +github.com/sapcc/go-bits v0.0.0-20241212131355-30c23561fbfd/go.mod h1:DrcK3N8lISMoxhS+e3pnrSyRZl83OGyNJdohpk9hjHo= github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= diff --git a/internal/api/api_test.go b/internal/api/api_test.go index 21a021c6e..a959212fd 100644 --- a/internal/api/api_test.go +++ b/internal/api/api_test.go @@ -47,6 +47,10 @@ import ( "github.com/sapcc/limes/internal/test/plugins" ) +func TestMain(m *testing.M) { + easypg.WithTestDB(m, func() int { return m.Run() }) +} + // NOTE: MiB makes no sense for a deletion rate, but I want to test as many // combinations of "has unit or not", "has limit or not" and "has usage or not" // as possible diff --git a/internal/collector/shared_test.go b/internal/collector/shared_test.go index a51bd40a9..463216b5d 100644 --- a/internal/collector/shared_test.go +++ b/internal/collector/shared_test.go @@ -23,9 +23,15 @@ import ( "testing" "time" + "github.com/sapcc/go-bits/easypg" + "github.com/sapcc/limes/internal/test" ) +func TestMain(m *testing.M) { + easypg.WithTestDB(m, func() int { return m.Run() }) +} + func getCollector(t *testing.T, s test.Setup) Collector { return Collector{ Cluster: s.Cluster, diff --git a/internal/datamodel/quota_overrides_test.go b/internal/datamodel/quota_overrides_test.go index 336418a9c..bdf3fa1ae 100644 --- a/internal/datamodel/quota_overrides_test.go +++ b/internal/datamodel/quota_overrides_test.go @@ -24,11 +24,16 @@ import ( "github.com/sapcc/go-api-declarations/liquid" "github.com/sapcc/go-bits/assert" + "github.com/sapcc/go-bits/easypg" "github.com/sapcc/limes/internal/db" "github.com/sapcc/limes/internal/test" ) +func TestMain(m *testing.M) { + easypg.WithTestDB(m, func() int { return m.Run() }) +} + const ( testQuotaOverridesNoRenamingConfigYAML = ` availability_zones: [ az-one, az-two ] diff --git a/internal/db/connection.go b/internal/db/connection.go index 64fcd79ee..015e14539 100644 --- a/internal/db/connection.go +++ b/internal/db/connection.go @@ -20,10 +20,12 @@ package db import ( - "net/url" + "database/sql" "os" + "github.com/dlmiddlecote/sqlstats" gorp "github.com/go-gorp/gorp/v3" + "github.com/prometheus/client_golang/prometheus" "github.com/sapcc/go-api-declarations/bininfo" "github.com/sapcc/go-bits/easypg" @@ -31,8 +33,15 @@ import ( "github.com/sapcc/go-bits/sqlext" ) +// Configuration returns the easypg.Configuration object that func Init() needs to initialize the DB connection. +func Configuration() easypg.Configuration { + return easypg.Configuration{ + Migrations: sqlMigrations, + } +} + // Init initializes the connection to the database. -func Init() (*gorp.DbMap, error) { +func Init() (*sql.DB, error) { extraConnectionOptions := make(map[string]string) if bininfo.Component() == "limes-serve" { // the API seems to have issues with connections getting stuck in "idle in transaction" during high load, not sure yet why @@ -50,26 +59,22 @@ func Init() (*gorp.DbMap, error) { if err != nil { return nil, err } - return InitFromURL(dbURL) -} - -// InitFromURL is like Init, but takes an explicit URL. This is used to -// override the default database URL configuration in tests. -func InitFromURL(dbURL *url.URL) (*gorp.DbMap, error) { - db, err := easypg.Connect(easypg.Configuration{ - PostgresURL: dbURL, - Migrations: sqlMigrations, - }) + dbConn, err := easypg.Connect(dbURL, Configuration()) if err != nil { return nil, err } + prometheus.MustRegister(sqlstats.NewStatsCollector("limes", dbConn)) + return dbConn, nil +} +// InitORM wraps a database connection into a gorp.DbMap instance. +func InitORM(dbConn *sql.DB) *gorp.DbMap { // ensure that this process does not starve other Limes processes for DB connections - db.SetMaxOpenConns(16) + dbConn.SetMaxOpenConns(16) - dbMap := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}} + dbMap := &gorp.DbMap{Db: dbConn, Dialect: gorp.PostgresDialect{}} initGorp(dbMap) - return dbMap, nil + return dbMap } // Interface provides the common methods that both SQL connections and diff --git a/internal/test/setup.go b/internal/test/setup.go index 682f8fb9b..971c83883 100644 --- a/internal/test/setup.go +++ b/internal/test/setup.go @@ -21,7 +21,7 @@ package test import ( "context" "net/http" - "net/url" + "slices" "strings" "testing" "time" @@ -45,6 +45,7 @@ import ( ) type setupParams struct { + DBSetupOptions []easypg.TestSetupOption DBFixtureFile string ConfigYAML string APIBuilder func(*core.Cluster, *gorp.DbMap, gopherpolicy.Validator, audittools.Auditor, func() time.Time, func() string) httpapi.API @@ -58,7 +59,7 @@ type SetupOption func(*setupParams) // the SQL statements in the given file. func WithDBFixtureFile(file string) SetupOption { return func(params *setupParams) { - params.DBFixtureFile = file + params.DBSetupOptions = append(params.DBSetupOptions, easypg.LoadSQLFile(file)) } } @@ -117,7 +118,7 @@ func NewSetup(t *testing.T, opts ...SetupOption) Setup { var s Setup s.Ctx = context.Background() - s.DB = initDatabase(t, params.DBFixtureFile) + s.DB = initDatabase(t, params.DBSetupOptions) s.Cluster = initCluster(t, s.Ctx, params.ConfigYAML) s.Clock = mock.NewClock() s.Registry = prometheus.NewPedanticRegistry() @@ -163,44 +164,18 @@ var cleanupProjectCommitmentsQuery = sqlext.SimplifyWhitespace(` ) `) -func initDatabase(t *testing.T, fixtureFile string) *gorp.DbMap { - //nolint:errcheck - postgresURL, _ := url.Parse("postgres://postgres:postgres@localhost:54321/limes?sslmode=disable") - dbm, err := db.InitFromURL(postgresURL) - if err != nil { - t.Error(err) - t.Log("Try prepending ./testing/with-postgres-db.sh to your command.") - t.FailNow() - } - - // reset the DB contents, starting with project_commitments because the "ON DELETE RESTRICT" constraint - // demands a specific deletion strategy - for { - result, err := dbm.Exec(cleanupProjectCommitmentsQuery) - if err != nil { - t.Fatal(err) - } - rowCount, err := result.RowsAffected() - if err != nil { - t.Fatal(err) - } - if rowCount == 0 { - break - } - } - - // reset the DB contents and populate with initial resources if requested - easypg.ClearTables(t, dbm.Db, "cluster_capacitors", "cluster_services", "domains") // all other tables via "ON DELETE CASCADE" - if fixtureFile != "" { - easypg.ExecSQLFile(t, dbm.Db, fixtureFile) - } - easypg.ResetPrimaryKeys(t, dbm.Db, - "cluster_services", "cluster_resources", "cluster_az_resources", - "domains", "projects", "project_commitments", - "project_services", "project_resources", "project_az_resources", +func initDatabase(t *testing.T, extraOpts []easypg.TestSetupOption) *gorp.DbMap { + opts := append(slices.Clone(extraOpts), + // project_commitments needs a specialized cleanup strategy because of an "ON DELETE RESTRICT" constraint + easypg.ClearContentsWith(cleanupProjectCommitmentsQuery), + easypg.ClearTables("cluster_capacitors", "cluster_services", "domains"), + easypg.ResetPrimaryKeys( + "cluster_services", "cluster_resources", "cluster_az_resources", + "domains", "projects", "project_commitments", + "project_services", "project_resources", "project_az_resources", + ), ) - - return dbm + return db.InitORM(easypg.ConnectForTest(t, db.Configuration(), opts...)) } func initCluster(t *testing.T, ctx context.Context, configYAML string) *core.Cluster { diff --git a/main.go b/main.go index b53782774..124d1c802 100644 --- a/main.go +++ b/main.go @@ -32,7 +32,6 @@ import ( "strings" "time" - "github.com/dlmiddlecote/sqlstats" "github.com/gophercloud/gophercloud/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" @@ -198,8 +197,7 @@ func taskCollect(ctx context.Context, cluster *core.Cluster, args []string) { isAuthoritative := osext.GetenvBool("LIMES_AUTHORITATIVE") // connect to database - dbm := must.Return(db.Init()) - prometheus.MustRegister(sqlstats.NewStatsCollector("limes", dbm.Db)) + dbm := db.InitORM(must.Return(db.Init())) // start scraping threads (NOTE: Many people use a pair of sync.WaitGroup and // stop channel to shutdown threads in a controlled manner. I decided against @@ -250,8 +248,7 @@ func taskServe(ctx context.Context, cluster *core.Cluster, args []string, provid } // connect to database - dbm := must.Return(db.Init()) - prometheus.MustRegister(sqlstats.NewStatsCollector("limes", dbm.Db)) + dbm := db.InitORM(must.Return(db.Init())) // connect to Hermes RabbitMQ if requested auditor := audittools.NewNullAuditor() @@ -296,8 +293,7 @@ func taskServeDataMetrics(ctx context.Context, cluster *core.Cluster, args []str } // connect to database - dbm := must.Return(db.Init()) - prometheus.MustRegister(sqlstats.NewStatsCollector("limes", dbm.Db)) + dbm := db.InitORM(must.Return(db.Init())) // serve data metrics skipZero := osext.GetenvBool("LIMES_DATA_METRICS_SKIP_ZERO") diff --git a/vendor/github.com/gophercloud/gophercloud/v2/CHANGELOG.md b/vendor/github.com/gophercloud/gophercloud/v2/CHANGELOG.md index efe2d9f6a..2520ac743 100644 --- a/vendor/github.com/gophercloud/gophercloud/v2/CHANGELOG.md +++ b/vendor/github.com/gophercloud/gophercloud/v2/CHANGELOG.md @@ -1,3 +1,11 @@ +## v2.3.0 (2024-12-06) + +* [GH-3213](https://github.com/gophercloud/gophercloud/pull/3213) [v2] Handle nova api version > 2.87 for hypervisor +* [GH-3236](https://github.com/gophercloud/gophercloud/pull/3236) [v2] Added required fields for Node API Parity +* [GH-3248](https://github.com/gophercloud/gophercloud/pull/3248) [v2] Add support for disable_power_off +* [GH-3261](https://github.com/gophercloud/gophercloud/pull/3261) [v2] Misc cleanups +* [GH-3262](https://github.com/gophercloud/gophercloud/pull/3262) [v2] Address govet 1.24 issue + ## v2.2.0 (2024-10-18) * [GH-3176](https://github.com/gophercloud/gophercloud/pull/3176) [v2] [containerinfra]: add "MasterLBEnabled" in Cluster results diff --git a/vendor/github.com/gophercloud/gophercloud/v2/Makefile b/vendor/github.com/gophercloud/gophercloud/v2/Makefile index 128beec00..2a0618a6b 100644 --- a/vendor/github.com/gophercloud/gophercloud/v2/Makefile +++ b/vendor/github.com/gophercloud/gophercloud/v2/Makefile @@ -1,6 +1,7 @@ undefine GOFLAGS -GOLANGCI_LINT_VERSION?=v1.57.1 +GOLANGCI_LINT_VERSION?=v1.62.2 +GO_TEST?=go run gotest.tools/gotestsum@latest --format testname -- ifeq ($(shell command -v podman 2> /dev/null),) RUNNER=docker @@ -22,88 +23,92 @@ lint: -v ~/.cache/golangci-lint/$(GOLANGCI_LINT_VERSION):/root/.cache \ -w /app \ -e GOFLAGS="-tags=acceptance" \ - golangci/golangci-lint:$(GOLANGCI_LINT_VERSION) golangci-lint run + golangci/golangci-lint:$(GOLANGCI_LINT_VERSION) golangci-lint run -v --max-same-issues 50 .PHONY: lint +format: + gofmt -w -s $(shell pwd) +.PHONY: format + unit: - go test ./... + $(GO_TEST) ./... .PHONY: unit coverage: - go test -covermode count -coverprofile cover.out -coverpkg=./... ./... + $(GO_TEST) -covermode count -coverprofile cover.out -coverpkg=./... ./... .PHONY: coverage acceptance: acceptance-baremetal acceptance-blockstorage acceptance-compute acceptance-container acceptance-containerinfra acceptance-db acceptance-dns acceptance-identity acceptance-imageservice acceptance-keymanager acceptance-loadbalancer acceptance-messaging acceptance-networking acceptance-objectstorage acceptance-orchestration acceptance-placement acceptance-sharedfilesystems acceptance-workflow .PHONY: acceptance acceptance-baremetal: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/baremetal/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/baremetal/... .PHONY: acceptance-baremetal acceptance-blockstorage: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/blockstorage/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/blockstorage/... .PHONY: acceptance-blockstorage acceptance-compute: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/compute/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/compute/... .PHONY: acceptance-compute acceptance-container: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/container/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/container/... .PHONY: acceptance-container acceptance-containerinfra: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/containerinfra/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/containerinfra/... .PHONY: acceptance-containerinfra acceptance-db: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/db/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/db/... .PHONY: acceptance-db acceptance-dns: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/dns/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/dns/... .PHONY: acceptance-dns acceptance-identity: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/identity/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/identity/... .PHONY: acceptance-identity acceptance-image: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/imageservice/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/imageservice/... .PHONY: acceptance-image acceptance-keymanager: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/keymanager/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/keymanager/... .PHONY: acceptance-keymanager acceptance-loadbalancer: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/loadbalancer/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/loadbalancer/... .PHONY: acceptance-loadbalancer acceptance-messaging: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/messaging/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/messaging/... .PHONY: acceptance-messaging acceptance-networking: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/networking/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/networking/... .PHONY: acceptance-networking acceptance-objectstorage: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/objectstorage/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/objectstorage/... .PHONY: acceptance-objectstorage acceptance-orchestration: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/orchestration/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/orchestration/... .PHONY: acceptance-orchestration acceptance-placement: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/placement/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/placement/... .PHONY: acceptance-placement acceptance-sharedfilesystems: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/sharedfilesystems/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/sharedfilesystems/... .PHONY: acceptance-sharefilesystems acceptance-workflow: - go test -tags "fixtures acceptance" ./internal/acceptance/openstack/workflow/... + $(GO_TEST) -tags "fixtures acceptance" ./internal/acceptance/openstack/workflow/... .PHONY: acceptance-workflow diff --git a/vendor/github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes/requests.go b/vendor/github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes/requests.go index 8cb0de9e0..8b2eb4b21 100644 --- a/vendor/github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes/requests.go +++ b/vendor/github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes/requests.go @@ -274,6 +274,10 @@ type CreateOpts struct { // Static network configuration to use during deployment and cleaning. NetworkData map[string]any `json:"network_data,omitempty"` + + // Whether disable_power_off is enabled or disabled on this node. + // Requires microversion 1.95 or later. + DisablePowerOff *bool `json:"disable_power_off,omitempty"` } // ToNodeCreateMap assembles a request body based on the contents of a CreateOpts. diff --git a/vendor/github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes/results.go b/vendor/github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes/results.go index 5c6eb5c19..507e0d6a4 100644 --- a/vendor/github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes/results.go +++ b/vendor/github.com/gophercloud/gophercloud/v2/openstack/baremetal/v1/nodes/results.go @@ -90,12 +90,19 @@ func (r SubscriptionVendorPassthruResult) Extract() (*SubscriptionVendorPassthru return &s, err } +// Link represents a hyperlink and its relationship to the current resource. +type Link struct { + // Href is the URL of the related resource. + Href string `json:"href"` + + // Rel describes the relationship of the resource to the current + // context (e.g., "self", "bookmark"). + Rel string `json:"rel"` +} + // Node represents a node in the OpenStack Bare Metal API. +// https://docs.openstack.org/api-ref/baremetal/#list-nodes-detailed type Node struct { - // Whether automated cleaning is enabled or disabled on this node. - // Requires microversion 1.47 or later. - AutomatedClean *bool `json:"automated_clean"` - // UUID for the resource. UUID string `json:"uuid"` @@ -183,8 +190,17 @@ type Node struct { // Current deploy step. DeployStep map[string]any `json:"deploy_step"` - // Current service step. - ServiceStep map[string]any `json:"service_step"` + // A list of relative links. Includes the self and bookmark links. + Links []Link `json:"links"` + + // Links to the collection of ports on this node + Ports []Link `json:"ports"` + + // Links to the collection of portgroups on this node. + PortGroups []Link `json:"portgroups"` + + // Links to the collection of states. Note that this resource is also used to request state transitions. + States []Link `json:"states"` // String which can be used by external schedulers to identify this Node as a unit of a specific type of resource. // For more details, see: https://docs.openstack.org/ironic/latest/install/configure-nova-flavors.html @@ -202,9 +218,6 @@ type Node struct { // Deploy interface for a node, e.g. “iscsi”. DeployInterface string `json:"deploy_interface"` - // Firmware interface for a node, e.g. “redfish”. - FirmwareInterface string `json:"firmware_interface"` - // Interface used for node inspection, e.g. “no-inspect”. InspectInterface string `json:"inspect_interface"` @@ -232,9 +245,15 @@ type Node struct { // For vendor-specific functionality on this node, e.g. “no-vendor”. VendorInterface string `json:"vendor_interface"` + // Links to the volume resources. + Volume []Link `json:"volume"` + // Conductor group for a node. Case-insensitive string up to 255 characters, containing a-z, 0-9, _, -, and .. ConductorGroup string `json:"conductor_group"` + // An optional UUID which can be used to denote the “parent” baremetal node. + ParentNode string `json:"parent_node"` + // The node is protected from undeploying, rebuilding and deletion. Protected bool `json:"protected"` @@ -244,14 +263,42 @@ type Node struct { // A string or UUID of the tenant who owns the baremetal node. Owner string `json:"owner"` + // A string or UUID of the tenant who is leasing the object. + Lessee string `json:"lessee"` + + // A string indicating the shard this node belongs to. + Shard string `json:"shard"` + + // Informational text about this node. + Description string `json:"description"` + + // The conductor currently servicing a node. This field is read-only. + Conductor string `json:"conductor"` + + // The UUID of the allocation associated with the node. If not null, will be the same as instance_uuid + // (the opposite is not always true). Unlike instance_uuid, this field is read-only. Please use the + // Allocation API to remove allocations. + AllocationUUID string `json:"allocation_uuid"` + + // Whether the node is retired. A Node tagged as retired will prevent any further + // scheduling of instances, but will still allow for other operations, such as cleaning, to happen + Retired bool `json:"retired"` + + // Reason the node is marked as retired. + RetiredReason string `json:"retired_reason"` + // Static network configuration to use during deployment and cleaning. NetworkData map[string]any `json:"network_data"` - // The UTC date and time when the resource was created, ISO 8601 format. - CreatedAt time.Time `json:"created_at"` + // Whether automated cleaning is enabled or disabled on this node. + // Requires microversion 1.47 or later. + AutomatedClean *bool `json:"automated_clean"` - // The UTC date and time when the resource was updated, ISO 8601 format. May be “null”. - UpdatedAt time.Time `json:"updated_at"` + // Current service step. + ServiceStep map[string]any `json:"service_step"` + + // Firmware interface for a node, e.g. “redfish”. + FirmwareInterface string `json:"firmware_interface"` // The UTC date and time when the provision state was updated, ISO 8601 format. May be “null”. ProvisionUpdatedAt time.Time `json:"provision_updated_at"` @@ -262,12 +309,15 @@ type Node struct { // The UTC date and time when the last inspection was finished, ISO 8601 format. May be “null” if inspection hasn't been finished yet. InspectionFinishedAt *time.Time `json:"inspection_finished_at"` - // Whether the node is retired. A Node tagged as retired will prevent any further - // scheduling of instances, but will still allow for other operations, such as cleaning, to happen - Retired bool `json:"retired"` + // The UTC date and time when the resource was created, ISO 8601 format. + CreatedAt time.Time `json:"created_at"` - // Reason the node is marked as retired. - RetiredReason string `json:"retired_reason"` + // The UTC date and time when the resource was updated, ISO 8601 format. May be “null”. + UpdatedAt time.Time `json:"updated_at"` + + // Whether disable_power_off is enabled or disabled on this node. + // Requires microversion 1.95 or later. + DisablePowerOff bool `json:"disable_power_off"` } // NodePage abstracts the raw results of making a List() request against diff --git a/vendor/github.com/gophercloud/gophercloud/v2/openstack/compute/v2/hypervisors/results.go b/vendor/github.com/gophercloud/gophercloud/v2/openstack/compute/v2/hypervisors/results.go index 2e8d24476..ee6046c6c 100644 --- a/vendor/github.com/gophercloud/gophercloud/v2/openstack/compute/v2/hypervisors/results.go +++ b/vendor/github.com/gophercloud/gophercloud/v2/openstack/compute/v2/hypervisors/results.go @@ -158,27 +158,31 @@ func (r *Hypervisor) UnmarshalJSON(b []byte) error { *r = Hypervisor(s.tmp) - // Newer versions return the CPU info as the correct type. - // Older versions return the CPU info as a string and need to be - // unmarshalled by the json parser. - var tmpb []byte - - switch t := s.CPUInfo.(type) { - case string: - tmpb = []byte(t) - case map[string]any: - tmpb, err = json.Marshal(t) - if err != nil { - return err + // cpu_info doesn't exist after api version 2.87, + // see https://docs.openstack.org/api-ref/compute/#id288 + if s.CPUInfo != nil { + // api versions 2.28 to 2.87 return the CPU info as the correct type. + // api versions < 2.28 return the CPU info as a string and need to be + // unmarshalled by the json parser. + var tmpb []byte + + switch t := s.CPUInfo.(type) { + case string: + tmpb = []byte(t) + case map[string]any: + tmpb, err = json.Marshal(t) + if err != nil { + return err + } + default: + return fmt.Errorf("CPUInfo has unexpected type: %T", t) } - default: - return fmt.Errorf("CPUInfo has unexpected type: %T", t) - } - if len(tmpb) != 0 { - err = json.Unmarshal(tmpb, &r.CPUInfo) - if err != nil { - return err + if len(tmpb) != 0 { + err = json.Unmarshal(tmpb, &r.CPUInfo) + if err != nil { + return err + } } } @@ -193,22 +197,28 @@ func (r *Hypervisor) UnmarshalJSON(b []byte) error { return fmt.Errorf("Hypervisor version has unexpected type: %T", t) } - switch t := s.FreeDiskGB.(type) { - case int: - r.FreeDiskGB = t - case float64: - r.FreeDiskGB = int(t) - default: - return fmt.Errorf("Free disk GB has unexpected type: %T", t) + // free_disk_gb doesn't exist after api version 2.87 + if s.FreeDiskGB != nil { + switch t := s.FreeDiskGB.(type) { + case int: + r.FreeDiskGB = t + case float64: + r.FreeDiskGB = int(t) + default: + return fmt.Errorf("Free disk GB has unexpected type: %T", t) + } } - switch t := s.LocalGB.(type) { - case int: - r.LocalGB = t - case float64: - r.LocalGB = int(t) - default: - return fmt.Errorf("Local GB has unexpected type: %T", t) + // local_gb doesn't exist after api version 2.87 + if s.LocalGB != nil { + switch t := s.LocalGB.(type) { + case int: + r.LocalGB = t + case float64: + r.LocalGB = int(t) + default: + return fmt.Errorf("Local GB has unexpected type: %T", t) + } } // OpenStack Compute service returns ID in string representation since diff --git a/vendor/github.com/gophercloud/gophercloud/v2/provider_client.go b/vendor/github.com/gophercloud/gophercloud/v2/provider_client.go index 007266f63..26c925cef 100644 --- a/vendor/github.com/gophercloud/gophercloud/v2/provider_client.go +++ b/vendor/github.com/gophercloud/gophercloud/v2/provider_client.go @@ -13,7 +13,7 @@ import ( // DefaultUserAgent is the default User-Agent string set in the request header. const ( - DefaultUserAgent = "gophercloud/v2.2.0" + DefaultUserAgent = "gophercloud/v2.3.0" DefaultMaxBackoffRetries = 60 ) diff --git a/vendor/github.com/sapcc/go-bits/easypg/easypg.go b/vendor/github.com/sapcc/go-bits/easypg/easypg.go index 49be05db6..785a866ff 100644 --- a/vendor/github.com/sapcc/go-bits/easypg/easypg.go +++ b/vendor/github.com/sapcc/go-bits/easypg/easypg.go @@ -26,7 +26,7 @@ import ( "database/sql" "errors" "fmt" - net_url "net/url" + url "net/url" "os" "regexp" "strings" @@ -58,9 +58,6 @@ import ( // `, // } type Configuration struct { - // (required) A libpq connection URL, see: - // - PostgresURL *net_url.URL // (required) The schema migrations, in Postgres syntax. See above for details. Migrations map[string]string // (optional) If not empty, use this database/sql driver instead of "postgres". @@ -68,14 +65,13 @@ type Configuration struct { OverrideDriverName string } -var errNoPostgresURL = errors.New("no PostgresURL given") - // Connect connects to a Postgres database. -func Connect(cfg Configuration) (*sql.DB, error) { - if cfg.PostgresURL == nil { - return nil, errNoPostgresURL - } - +// +// The given URL must be a libpq connection URL, see: +// +// +// We recommend constructing the URL with func URLFrom. +func Connect(dbURL url.URL, cfg Configuration) (*sql.DB, error) { migrations := cfg.Migrations migrations = wrapDDLInTransactions(migrations) migrations = stripWhitespace(migrations) @@ -98,7 +94,7 @@ func Connect(cfg Configuration) (*sql.DB, error) { return nil, err } - db, dbd, err := connectToPostgres(cfg.PostgresURL, cfg.OverrideDriverName) + db, dbd, err := connectToPostgres(dbURL, cfg.OverrideDriverName) if err != nil { return nil, fmt.Errorf("cannot connect to Postgres: %w", err) } @@ -112,11 +108,11 @@ func Connect(cfg Configuration) (*sql.DB, error) { var dbNotExistErrRx = regexp.MustCompile(`^pq: database "([^"]+)" does not exist$`) -func connectToPostgres(url *net_url.URL, driverName string) (*sql.DB, database.Driver, error) { +func connectToPostgres(dbURL url.URL, driverName string) (*sql.DB, database.Driver, error) { if driverName == "" { driverName = "postgres" } - db, err := sql.Open(driverName, url.String()) + db, err := sql.Open(driverName, dbURL.String()) if err == nil { // apparently the "database does not exist" error only occurs when trying to issue the first statement _, err = db.Exec("SELECT 1") @@ -135,7 +131,7 @@ func connectToPostgres(url *net_url.URL, driverName string) (*sql.DB, database.D // connect to Postgres without the database name specified, so that we can // execute CREATE DATABASE - urlWithoutDB := *url + urlWithoutDB := dbURL urlWithoutDB.Path = "/" db2, err := sql.Open(driverName, urlWithoutDB.String()) if err == nil { @@ -151,7 +147,7 @@ func connectToPostgres(url *net_url.URL, driverName string) (*sql.DB, database.D } // now the actual database is there and we can connect to it - db, err = sql.Open(driverName, url.String()) + db, err = sql.Open(driverName, dbURL.String()) if err != nil { return nil, nil, err } diff --git a/vendor/github.com/sapcc/go-bits/easypg/testhelpers.go b/vendor/github.com/sapcc/go-bits/easypg/testhelpers.go index ed74f8ef8..e6839a73f 100644 --- a/vendor/github.com/sapcc/go-bits/easypg/testhelpers.go +++ b/vendor/github.com/sapcc/go-bits/easypg/testhelpers.go @@ -32,60 +32,6 @@ import ( "github.com/sapcc/go-bits/osext" ) -// ClearTables removes all rows from the given tables. -func ClearTables(t *testing.T, db *sql.DB, tableNames ...string) { - t.Helper() - for _, tableName := range tableNames { - _, err := db.Exec("DELETE FROM " + tableName) //nolint:gosec // cannot provide tableName as bind parameter - if err != nil { - t.Fatalf("while clearing table %s: %s", tableName, err.Error()) - } - } -} - -// ResetPrimaryKeys resets the sequences for the "id" column of the given tables -// to start at 1 again (or if there are entries in the table, to start right -// after the entry with the highest ID). -func ResetPrimaryKeys(t *testing.T, db *sql.DB, tableNames ...string) { - t.Helper() - for _, tableName := range tableNames { - var nextID int64 - query := "SELECT 1 + COALESCE(MAX(id), 0) FROM " + tableName //nolint:gosec // cannot provide tableName as bind parameter - err := db.QueryRow(query).Scan(&nextID) - if err != nil { - t.Fatalf("while checking IDs in table %s: %s", tableName, err.Error()) - } - - query = fmt.Sprintf(`ALTER SEQUENCE %s_id_seq RESTART WITH %d`, tableName, nextID) - _, err = db.Exec(query) - if err != nil { - t.Fatalf("while resetting ID sequence on table %s: %s", tableName, err.Error()) - } - } -} - -// ExecSQLFile loads a file containing SQL statements and executes them all. -// It implies that every SQL statement is on a single line. -func ExecSQLFile(t *testing.T, db *sql.DB, path string) { - t.Helper() - sqlBytes, err := os.ReadFile(path) - if err != nil { - t.Fatal(err) - } - - // split into single statements because db.Exec() will just ignore everything after the first semicolon - for idx, line := range strings.Split(string(sqlBytes), "\n") { - line = strings.TrimSpace(line) - if line == "" || strings.HasPrefix(line, "--") { - continue - } - _, err = db.Exec(line) - if err != nil { - t.Fatalf("error on SQL line %d: %s", idx, err.Error()) - } - } -} - // AssertDBContent makes a dump of the database contents (as a sequence of // INSERT statements) and runs diff(1) against the given file, producing a test // error if these two are different from each other. diff --git a/vendor/github.com/sapcc/go-bits/easypg/testsetup.go b/vendor/github.com/sapcc/go-bits/easypg/testsetup.go new file mode 100644 index 000000000..c47a0c418 --- /dev/null +++ b/vendor/github.com/sapcc/go-bits/easypg/testsetup.go @@ -0,0 +1,332 @@ +/****************************************************************************** +* +* Copyright 2024 SAP SE +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +* +******************************************************************************/ + +package easypg + +import ( + "database/sql" + "errors" + "fmt" + url "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/sapcc/go-bits/logg" + "github.com/sapcc/go-bits/must" + "github.com/sapcc/go-bits/sqlext" +) + +// this custom port avoids conflicts with any system-wide Postgres instances on the standard port 5432 +const testDBPort = 54320 + +var clientLaunchScript = `#!/usr/bin/env bash +set -euo pipefail + +stop_postgres() { + EXIT_CODE=$? + pg_ctl stop --wait --silent -D .testdb/datadir + exit "${EXIT_CODE}" +} +trap stop_postgres EXIT INT TERM + +rm -f -- .testdb/run/postgresql.log +pg_ctl start --wait --silent -D .testdb/datadir -l .testdb/run/postgresql.log +%[1]s -U postgres -h 127.0.0.1 -p %[2]d "$@" +` + +var hasTestDB = false + +// WithTestDB spawns a PostgreSQL database for the duration of a `go test` run. +// Its data directory, configuration and logs are stored in the ".testdb" directory below the repository root. +// +// How to interact with the test database: +// - To inspect it manually, use one of the helper scripts in the ".testdb" directory, e.g. ".testdb/psql.sh". +// - It is currently not supported to run tests for multiple packages concurrently, so make sure to run "go test" with "-p 1". +// - The "/.testdb" directory should be added to your repository's .gitignore rules. +// +// This function takes a testing.M because it is supposed to be called from TestMain(). +// This is required to ensure that its cleanup phase shuts down the database server after all tests have been executed. +// Add a TestMain() like this to each package that needs to interact with the test database: +// +// func TestMain(m *testing.M) { +// easypg.WithTestDB(m, func() int { return m.Run() }) +// } +// +// This function will fail when running as root (which might happen in some Docker containers), because PostgreSQL refuses to run as UID 0. +func WithTestDB(m *testing.M, action func() int) int { + rootPath := must.Return(findRepositoryRootDir()) + + // create DB on first use + hasPostgresDB := must.Return(checkPathExists(filepath.Join(rootPath, ".testdb/datadir/PG_VERSION"))) + if !hasPostgresDB { + for _, dirName := range []string{".testdb/datadir", ".testdb/run"} { + must.Succeed(os.MkdirAll(filepath.Join(rootPath, dirName), 0777)) // subject to umask + } + cmd := exec.Command("initdb", "-A", "trust", "-U", "postgres", //nolint:gosec // rule G204 is overly broad + "-D", filepath.Join(rootPath, ".testdb/datadir"), + "-c", "external_pid_file="+filepath.Join(rootPath, ".testdb/run/pid"), + "-c", "unix_socket_directories="+filepath.Join(rootPath, ".testdb/run"), + "-c", fmt.Sprintf("port=%d", testDBPort), + ) + cmd.Stdin = nil + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + logg.Fatal("could not run initdb: %s", err.Error()) + } + } + + // drop helper scripts that can be used to attach to the test DB for manual debugging and inspection + for _, clientTool := range []string{"psql", "pgcli", "pg_dump"} { + path := filepath.Join(rootPath, ".testdb", clientTool+".sh") + contents := fmt.Sprintf(clientLaunchScript, clientTool, testDBPort) + must.Succeed(os.WriteFile(path, []byte(contents), 0777)) // subject to umask, intentionally executable + } + + // start database process + cmd := exec.Command("pg_ctl", "start", "--wait", "--silent", //nolint:gosec // rule G204 is overly broad + "-D", filepath.Join(rootPath, ".testdb/datadir"), + "-l", filepath.Join(rootPath, ".testdb/run/postgresql.log"), + ) + cmd.Stdin = nil + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err := cmd.Run() + if err != nil { + logg.Fatal("could not run pg_ctl start: %s", err.Error()) + } + + // run tests + hasTestDB = true + exitCode := action() + hasTestDB = false + + // stop database process (regardless of whether tests succeeded or failed!) + cmd = exec.Command("pg_ctl", "stop", "--wait", "--silent", //nolint:gosec // rule G204 is overly broad + "-D", filepath.Join(rootPath, ".testdb/datadir"), + ) + cmd.Stdin = nil + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + err = cmd.Run() + if err != nil { + logg.Fatal("could not run pg_ctl stop: %s", err.Error()) + } + + return exitCode +} + +func findRepositoryRootDir() (string, error) { + // NOTE: `go test` runs each test within the directory containing its source code. + dirPath, err := os.Getwd() + if err != nil { + return "", err + } + for { + isRepoRoot, err := checkPathExists(filepath.Join(dirPath, "go.mod")) + switch { + case err != nil: + return "", err + case isRepoRoot: + return dirPath, nil + default: + // this is not the repo root, keep searching + parentPath := filepath.Dir(dirPath) + if parentPath == dirPath { + return "", errors.New("could not find repository root (neither $PWD nor any parents contain a go.mod file)") + } + dirPath = parentPath + } + } +} + +func checkPathExists(path string) (bool, error) { + _, err := os.Stat(path) + switch { + case err == nil: + return true, nil + case os.IsNotExist(err): + return false, nil + default: + return false, err + } +} + +type testSetupParams struct { + databaseName string + sqlStatementsForClear []string + tableNamesForClear []string + sqlFileToLoad string + tableNamesForPKReset []string +} + +// TestSetupOption is an optional behavior that can be given to ConnectForTest(). +type TestSetupOption func(*testSetupParams) + +// ClearContentsWith is a TestSetupOption that removes records from the DB using the provided SQL statement. +// If provided, this runs directly after connecting, before any other setup phase. +// The provided SQL statement is executed repeatedly, until result.RowsAffected() == 0 is observed. +// +// Prefer ClearTables() over this, and only use this if ClearTables() does not work. +func ClearContentsWith(sqlStatement string) TestSetupOption { + return func(params *testSetupParams) { + params.sqlStatementsForClear = append(params.sqlStatementsForClear, sqlext.SimplifyWhitespace(sqlStatement)) + } +} + +// ClearTables is a TestSetupOption that removes all rows from the given tables. +// +// This option only works for tables that can be cleared with `DELETE FROM `. +// If specific setups like "ON DELETE RESTRICT" constraints make that impossible, +// use ClearContentsWith() to provide a specialized clearing method that runs before ClearTables(). +func ClearTables(tableNames ...string) TestSetupOption { + return func(params *testSetupParams) { + params.tableNamesForClear = append(params.tableNamesForClear, tableNames...) + } +} + +// LoadSQLFile is a TestSetupOption that loads a file containing SQL statements and executes them all. +// Every SQL statement must be on a single line. +// +// This executes after any ClearTables() options, but before any ResetPrimaryKeys() options. +func LoadSQLFile(path string) TestSetupOption { + return func(params *testSetupParams) { + params.sqlFileToLoad = path + } +} + +// ResetPrimaryKeys is a TestSetupOption that resets the sequences for the "id" +// column of the given tables to start at 1 again (or if there are entries in +// the table, to start right after the entry with the highest ID). +func ResetPrimaryKeys(tableNames ...string) TestSetupOption { + return func(params *testSetupParams) { + params.tableNamesForPKReset = append(params.tableNamesForPKReset, tableNames...) + } +} + +// OverrideDatabaseName is a TestSetupOption that picks a different database +// name than the default of t.Name(). +// +// This is only necessary if a single test needs to use multiple database connections at the same time, +// e.g. to simulate two separate deployments of the application next to each other. +func OverrideDatabaseName(dbName string) TestSetupOption { + return func(params *testSetupParams) { + params.databaseName = dbName + } +} + +// ConnectForTest connects to the test database server managed by func WithTestDB(). +// Any number of TestSetupOption arguments can be given to reset and prepare the database for the test run. +// +// Each test will run in its own separate database (whose name is the same as the test name), +// so it is safe to mark tests as t.Parallel() to run multiple tests within the same package concurrently. +func ConnectForTest(t *testing.T, cfg Configuration, opts ...TestSetupOption) *sql.DB { + t.Helper() + + var params testSetupParams + for _, o := range opts { + o(¶ms) + } + + // input validation + if !hasTestDB { + t.Fatal("easypg.ConnectForTest() can only be used if easypg.WithTestDB() was called in TestMain (see docs on func WithTestDB for details)") + } + + // connect to DB (the database name is set to the test name to isolate concurrent tests from each other) + dbName := t.Name() + if params.databaseName != "" { + dbName = params.databaseName + } + dbURLStr := fmt.Sprintf("postgres://postgres:postgres@127.0.0.1:%d/%s?sslmode=disable", testDBPort, strings.ToLower(dbName)) + dbURL, err := url.Parse(dbURLStr) + if err != nil { + t.Fatalf("malformed database URL %q: %s", dbURLStr, err.Error()) + } + db, err := Connect(*dbURL, cfg) + if err != nil { + t.Fatal(err.Error()) + } + + // execute ClearContentsWith() setup options, if any + for _, sqlStatement := range params.sqlStatementsForClear { + for { + result, err := db.Exec(sqlStatement) + if err != nil { + t.Fatalf("while clearing contents with %q: %s", sqlStatement, err.Error()) + } + rowCount, err := result.RowsAffected() + if err != nil { + t.Fatalf("while clearing contents with %q: %s", sqlStatement, err.Error()) + } + if rowCount == 0 { + break + } + } + } + + // execute ClearTables() setup option, if any + for _, tableName := range params.tableNamesForClear { + _, err := db.Exec(fmt.Sprintf(`DELETE FROM "%s"`, tableName)) + if err != nil { + t.Fatalf("while clearing table %s: %s", tableName, err.Error()) + } + } + + // execute ExecSQLFile() setup option, if any + if params.sqlFileToLoad != "" { + sqlBytes, err := os.ReadFile(params.sqlFileToLoad) + if err != nil { + t.Fatal(err.Error()) + } + + // split into single statements because db.Exec() will just ignore everything after the first semicolon + for idx, line := range strings.Split(string(sqlBytes), "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "--") { + continue + } + _, err = db.Exec(line) + if err != nil { + t.Fatalf("error in %s on line %d: %s", params.sqlFileToLoad, idx, err.Error()) + } + } + } + + // execute ResetPrimaryKeys() setup option, if any + for _, tableName := range params.tableNamesForPKReset { + var nextID int64 + query := fmt.Sprintf(`SELECT 1 + COALESCE(MAX(id), 0) FROM "%s"`, tableName) //nolint:gosec // we are just using it for tests + err := db.QueryRow(query).Scan(&nextID) + if err != nil { + t.Fatalf("while checking IDs in table %s: %s", tableName, err.Error()) + } + + query = fmt.Sprintf(`ALTER SEQUENCE %s_id_seq RESTART WITH %d`, tableName, nextID) + _, err = db.Exec(query) + if err != nil { + t.Fatalf("while resetting ID sequence on table %s: %s", tableName, err.Error()) + } + } + + return db +} diff --git a/vendor/github.com/sapcc/go-bits/easypg/url.go b/vendor/github.com/sapcc/go-bits/easypg/url.go index 87470eaf0..c59961abe 100644 --- a/vendor/github.com/sapcc/go-bits/easypg/url.go +++ b/vendor/github.com/sapcc/go-bits/easypg/url.go @@ -46,23 +46,24 @@ var osHostname = os.Hostname // URLFrom constructs a libpq connection URL from the provided parts. The parts // are typically retrieved from environment variables, for example: // -// cfg.PostgresURL = easypg.URLFrom(easypg.URLParts { +// dbURL := must.Return(easypg.URLFrom(easypg.URLParts { // HostName: osext.GetenvOrDefault("FOOBAR_DB_HOSTNAME", "localhost"), // Port: osext.GetenvOrDefault("FOOBAR_DB_PORT", "5432"), // UserName: osext.GetenvOrDefault("FOOBAR_DB_USERNAME", "postgres"), // Password: os.Getenv("FOOBAR_DB_PASSWORD"), // ConnectionOptions: os.Getenv("FOOBAR_DB_CONNECTION_OPTIONS"), // DatabaseName: osext.GetenvOrDefault("FOOBAR_DB_NAME", "foobar"), -// }) +// })) +// db := must.Return(easypg.Connect(dbURL, easypg.Configuration{ ... })) // // We provide URLFrom() as a separate function, instead of just putting the // fields of URLParts into the Configuration struct, to accommodate applications // that may want to accept a fully-formed postgres:// URL from outside instead // of building it up from individual parts. -func URLFrom(parts URLParts) (*url.URL, error) { +func URLFrom(parts URLParts) (url.URL, error) { connOpts, err := url.ParseQuery(parts.ConnectionOptions) if err != nil { - return nil, fmt.Errorf("cannot parse DB connection options (%q): %w", parts.ConnectionOptions, err) + return url.URL{}, fmt.Errorf("cannot parse DB connection options (%q): %w", parts.ConnectionOptions, err) } hostname, err := osHostname() @@ -92,5 +93,5 @@ func URLFrom(parts URLParts) (*url.URL, error) { result.Host = net.JoinHostPort(parts.HostName, parts.Port) } - return &result, nil + return result, nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index b56472eba..6dba6ca61 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -25,7 +25,7 @@ github.com/golang-migrate/migrate/v4/database/postgres github.com/golang-migrate/migrate/v4/internal/url github.com/golang-migrate/migrate/v4/source github.com/golang-migrate/migrate/v4/source/go_bindata -# github.com/gophercloud/gophercloud/v2 v2.2.0 +# github.com/gophercloud/gophercloud/v2 v2.3.0 ## explicit; go 1.22 github.com/gophercloud/gophercloud/v2 github.com/gophercloud/gophercloud/v2/openstack @@ -150,7 +150,7 @@ github.com/sapcc/go-api-declarations/limes github.com/sapcc/go-api-declarations/limes/rates github.com/sapcc/go-api-declarations/limes/resources github.com/sapcc/go-api-declarations/liquid -# github.com/sapcc/go-bits v0.0.0-20241206132118-f18a227dc952 +# github.com/sapcc/go-bits v0.0.0-20241212131355-30c23561fbfd ## explicit; go 1.23 github.com/sapcc/go-bits/assert github.com/sapcc/go-bits/audittools From 9f3989741ffccc1eab1c70ff2908c5dc6fc4fe44 Mon Sep 17 00:00:00 2001 From: Stefan Majewsky Date: Thu, 12 Dec 2024 14:23:50 +0100 Subject: [PATCH 2/2] fix sqlstats sometimes taking wrong DB name --- internal/db/connection.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/db/connection.go b/internal/db/connection.go index 015e14539..b66d8684d 100644 --- a/internal/db/connection.go +++ b/internal/db/connection.go @@ -48,13 +48,14 @@ func Init() (*sql.DB, error) { extraConnectionOptions["idle_in_transaction_session_timeout"] = "10000" // 10000 ms = 10 seconds } + dbName := osext.GetenvOrDefault("LIMES_DB_NAME", "limes") dbURL, err := easypg.URLFrom(easypg.URLParts{ HostName: osext.GetenvOrDefault("LIMES_DB_HOSTNAME", "localhost"), Port: osext.GetenvOrDefault("LIMES_DB_PORT", "5432"), UserName: osext.GetenvOrDefault("LIMES_DB_USERNAME", "postgres"), Password: os.Getenv("LIMES_DB_PASSWORD"), ConnectionOptions: os.Getenv("LIMES_DB_CONNECTION_OPTIONS"), - DatabaseName: osext.GetenvOrDefault("LIMES_DB_NAME", "limes"), + DatabaseName: dbName, }) if err != nil { return nil, err @@ -63,7 +64,7 @@ func Init() (*sql.DB, error) { if err != nil { return nil, err } - prometheus.MustRegister(sqlstats.NewStatsCollector("limes", dbConn)) + prometheus.MustRegister(sqlstats.NewStatsCollector(dbName, dbConn)) return dbConn, nil }