Merge pull request #624 from sapcc/update-easypg

use new go-bits/easypg API

.gitignore

@@ -2,7 +2,7 @@
 build/
 
 # test artifacts
-testing/postgres*
+/.testdb
 *.actual
 
 # custom configuration files used during development

go.mod

@@ -7,14 +7,15 @@ require (
 	github.com/dlmiddlecote/sqlstats v1.0.2
 	github.com/go-gorp/gorp/v3 v3.1.0
 	github.com/gofrs/uuid/v5 v5.3.0
-	github.com/gophercloud/gophercloud/v2 v2.2.0
+	github.com/gophercloud/gophercloud/v2 v2.3.0
 	github.com/gorilla/mux v1.8.1
+	github.com/lib/pq v1.10.9
 	github.com/majewsky/schwift/v2 v2.0.0
 	github.com/prometheus/client_golang v1.20.5
 	github.com/prometheus/common v0.61.0
 	github.com/rs/cors v1.11.1
 	github.com/sapcc/go-api-declarations v1.13.1
-	github.com/sapcc/go-bits v0.0.0-20241206132118-f18a227dc952
+	github.com/sapcc/go-bits v0.0.0-20241212131355-30c23561fbfd
 	go.uber.org/automaxprocs v1.6.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -29,7 +30,6 @@ require (
 	github.com/jpillora/longestcommon v0.0.0-20161227235612-adb9d91ee629 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
-	github.com/lib/pq v1.10.9 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect

go.sum

@@ -59,8 +59,8 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/gophercloud/gophercloud/v2 v2.2.0 h1:STqqnSXuhcg1OPBOZ14z6JDm8fKIN13H2bJg6bBuHp8=
-github.com/gophercloud/gophercloud/v2 v2.2.0/go.mod h1:f2hMRC7Kakbv5vM7wSGHrIPZh6JZR60GVHryJlF/K44=
+github.com/gophercloud/gophercloud/v2 v2.3.0 h1:5ipI2Mgxee0TwQxqnOIUdTbzL4ZBB8GORyZko+yGXI0=
+github.com/gophercloud/gophercloud/v2 v2.3.0/go.mod h1:uJWNpTgJPSl2gyzJqcU/pIAhFUWvIkp8eE8M15n9rs4=
 github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
 github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -159,8 +159,8 @@ github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA=
 github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU=
 github.com/sapcc/go-api-declarations v1.13.1 h1:rovCnLscnoZaIZPWhohSYHzwwYjOnCPsRw3zwtu4tLI=
 github.com/sapcc/go-api-declarations v1.13.1/go.mod h1:83R3hTANhuRXt/pXDby37IJetw8l7DG41s33Tp9NXxI=
-github.com/sapcc/go-bits v0.0.0-20241206132118-f18a227dc952 h1:tg1xF/eh6kM3Ti5AmMCV6zNKRPHtmP8Yh1XYG0/WbSk=
-github.com/sapcc/go-bits v0.0.0-20241206132118-f18a227dc952/go.mod h1:ROdTmzQj/gn6dUaxhrCQCTsZtaFJPAFy3CeTD2m/z3k=
+github.com/sapcc/go-bits v0.0.0-20241212131355-30c23561fbfd h1:w70x4iw6MjZwRXjFARt3xfDjBq/MunXRdToH+wvFvF8=
+github.com/sapcc/go-bits v0.0.0-20241212131355-30c23561fbfd/go.mod h1:DrcK3N8lISMoxhS+e3pnrSyRZl83OGyNJdohpk9hjHo=
 github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
 github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=

internal/api/api_test.go

@@ -47,6 +47,10 @@ import (
 	"github.com/sapcc/limes/internal/test/plugins"
 )
 
+func TestMain(m *testing.M) {
+	easypg.WithTestDB(m, func() int { return m.Run() })
+}
+
 // NOTE: MiB makes no sense for a deletion rate, but I want to test as many
 // combinations of "has unit or not", "has limit or not" and "has usage or not"
 // as possible

internal/collector/shared_test.go

@@ -23,9 +23,15 @@ import (
 	"testing"
 	"time"
 
+	"github.com/sapcc/go-bits/easypg"
+
 	"github.com/sapcc/limes/internal/test"
 )
 
+func TestMain(m *testing.M) {
+	easypg.WithTestDB(m, func() int { return m.Run() })
+}
+
 func getCollector(t *testing.T, s test.Setup) Collector {
 	return Collector{
 		Cluster:     s.Cluster,

internal/datamodel/quota_overrides_test.go

@@ -24,11 +24,16 @@ import (
 
 	"github.com/sapcc/go-api-declarations/liquid"
 	"github.com/sapcc/go-bits/assert"
+	"github.com/sapcc/go-bits/easypg"
 
 	"github.com/sapcc/limes/internal/db"
 	"github.com/sapcc/limes/internal/test"
 )
 
+func TestMain(m *testing.M) {
+	easypg.WithTestDB(m, func() int { return m.Run() })
+}
+
 const (
 	testQuotaOverridesNoRenamingConfigYAML = `
 		availability_zones: [ az-one, az-two ]

internal/db/connection.go

@@ -20,56 +20,62 @@
 package db
 
 import (
-	"net/url"
+	"database/sql"
 	"os"
 
+	"github.com/dlmiddlecote/sqlstats"
 	gorp "github.com/go-gorp/gorp/v3"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/sapcc/go-api-declarations/bininfo"
 	"github.com/sapcc/go-bits/easypg"
 	"github.com/sapcc/go-bits/osext"
 	"github.com/sapcc/go-bits/sqlext"
 )
 
+// Configuration returns the easypg.Configuration object that func Init() needs to initialize the DB connection.
+func Configuration() easypg.Configuration {
+	return easypg.Configuration{
+		Migrations: sqlMigrations,
+	}
+}
+
 // Init initializes the connection to the database.
-func Init() (*gorp.DbMap, error) {
+func Init() (*sql.DB, error) {
 	extraConnectionOptions := make(map[string]string)
 	if bininfo.Component() == "limes-serve" {
 		// the API seems to have issues with connections getting stuck in "idle in transaction" during high load, not sure yet why
 		extraConnectionOptions["idle_in_transaction_session_timeout"] = "10000" // 10000 ms = 10 seconds
 	}
 
+	dbName := osext.GetenvOrDefault("LIMES_DB_NAME", "limes")
 	dbURL, err := easypg.URLFrom(easypg.URLParts{
 		HostName:          osext.GetenvOrDefault("LIMES_DB_HOSTNAME", "localhost"),
 		Port:              osext.GetenvOrDefault("LIMES_DB_PORT", "5432"),
 		UserName:          osext.GetenvOrDefault("LIMES_DB_USERNAME", "postgres"),
 		Password:          os.Getenv("LIMES_DB_PASSWORD"),
 		ConnectionOptions: os.Getenv("LIMES_DB_CONNECTION_OPTIONS"),
-		DatabaseName:      osext.GetenvOrDefault("LIMES_DB_NAME", "limes"),
+		DatabaseName:      dbName,
 	})
 	if err != nil {
 		return nil, err
 	}
-	return InitFromURL(dbURL)
-}
-
-// InitFromURL is like Init, but takes an explicit URL. This is used to
-// override the default database URL configuration in tests.
-func InitFromURL(dbURL *url.URL) (*gorp.DbMap, error) {
-	db, err := easypg.Connect(easypg.Configuration{
-		PostgresURL: dbURL,
-		Migrations:  sqlMigrations,
-	})
+	dbConn, err := easypg.Connect(dbURL, Configuration())
 	if err != nil {
 		return nil, err
 	}
+	prometheus.MustRegister(sqlstats.NewStatsCollector(dbName, dbConn))
+	return dbConn, nil
+}
 
+// InitORM wraps a database connection into a gorp.DbMap instance.
+func InitORM(dbConn *sql.DB) *gorp.DbMap {
 	// ensure that this process does not starve other Limes processes for DB connections
-	db.SetMaxOpenConns(16)
+	dbConn.SetMaxOpenConns(16)
 
-	dbMap := &gorp.DbMap{Db: db, Dialect: gorp.PostgresDialect{}}
+	dbMap := &gorp.DbMap{Db: dbConn, Dialect: gorp.PostgresDialect{}}
 	initGorp(dbMap)
-	return dbMap, nil
+	return dbMap
 }
 
 // Interface provides the common methods that both SQL connections and

internal/test/setup.go

@@ -21,7 +21,7 @@ package test
 import (
 	"context"
 	"net/http"
-	"net/url"
+	"slices"
 	"strings"
 	"testing"
 	"time"
@@ -45,6 +45,7 @@ import (
 )
 
 type setupParams struct {
+	DBSetupOptions []easypg.TestSetupOption
 	DBFixtureFile  string
 	ConfigYAML     string
 	APIBuilder     func(*core.Cluster, *gorp.DbMap, gopherpolicy.Validator, audittools.Auditor, func() time.Time, func() string) httpapi.API
@@ -58,7 +59,7 @@ type SetupOption func(*setupParams)
 // the SQL statements in the given file.
 func WithDBFixtureFile(file string) SetupOption {
 	return func(params *setupParams) {
-		params.DBFixtureFile = file
+		params.DBSetupOptions = append(params.DBSetupOptions, easypg.LoadSQLFile(file))
 	}
 }
 
@@ -117,7 +118,7 @@ func NewSetup(t *testing.T, opts ...SetupOption) Setup {
 
 	var s Setup
 	s.Ctx = context.Background()
-	s.DB = initDatabase(t, params.DBFixtureFile)
+	s.DB = initDatabase(t, params.DBSetupOptions)
 	s.Cluster = initCluster(t, s.Ctx, params.ConfigYAML)
 	s.Clock = mock.NewClock()
 	s.Registry = prometheus.NewPedanticRegistry()
@@ -163,44 +164,18 @@ var cleanupProjectCommitmentsQuery = sqlext.SimplifyWhitespace(`
 	)
 `)
 
-func initDatabase(t *testing.T, fixtureFile string) *gorp.DbMap {
-	//nolint:errcheck
-	postgresURL, _ := url.Parse("postgres://postgres:postgres@localhost:54321/limes?sslmode=disable")
-	dbm, err := db.InitFromURL(postgresURL)
-	if err != nil {
-		t.Error(err)
-		t.Log("Try prepending ./testing/with-postgres-db.sh to your command.")
-		t.FailNow()
-	}
-
-	// reset the DB contents, starting with project_commitments because the "ON DELETE RESTRICT" constraint
-	// demands a specific deletion strategy
-	for {
-		result, err := dbm.Exec(cleanupProjectCommitmentsQuery)
-		if err != nil {
-			t.Fatal(err)
-		}
-		rowCount, err := result.RowsAffected()
-		if err != nil {
-			t.Fatal(err)
-		}
-		if rowCount == 0 {
-			break
-		}
-	}
-
-	// reset the DB contents and populate with initial resources if requested
-	easypg.ClearTables(t, dbm.Db, "cluster_capacitors", "cluster_services", "domains") // all other tables via "ON DELETE CASCADE"
-	if fixtureFile != "" {
-		easypg.ExecSQLFile(t, dbm.Db, fixtureFile)
-	}
-	easypg.ResetPrimaryKeys(t, dbm.Db,
-		"cluster_services", "cluster_resources", "cluster_az_resources",
-		"domains", "projects", "project_commitments",
-		"project_services", "project_resources", "project_az_resources",
+func initDatabase(t *testing.T, extraOpts []easypg.TestSetupOption) *gorp.DbMap {
+	opts := append(slices.Clone(extraOpts),
+		// project_commitments needs a specialized cleanup strategy because of an "ON DELETE RESTRICT" constraint
+		easypg.ClearContentsWith(cleanupProjectCommitmentsQuery),
+		easypg.ClearTables("cluster_capacitors", "cluster_services", "domains"),
+		easypg.ResetPrimaryKeys(
+			"cluster_services", "cluster_resources", "cluster_az_resources",
+			"domains", "projects", "project_commitments",
+			"project_services", "project_resources", "project_az_resources",
+		),
 	)
-
-	return dbm
+	return db.InitORM(easypg.ConnectForTest(t, db.Configuration(), opts...))
 }
 
 func initCluster(t *testing.T, ctx context.Context, configYAML string) *core.Cluster {

main.go

@@ -32,7 +32,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/dlmiddlecote/sqlstats"
 	"github.com/gophercloud/gophercloud/v2"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
@@ -198,8 +197,7 @@ func taskCollect(ctx context.Context, cluster *core.Cluster, args []string) {
 	isAuthoritative := osext.GetenvBool("LIMES_AUTHORITATIVE")
 
 	// connect to database
-	dbm := must.Return(db.Init())
-	prometheus.MustRegister(sqlstats.NewStatsCollector("limes", dbm.Db))
+	dbm := db.InitORM(must.Return(db.Init()))
 
 	// start scraping threads (NOTE: Many people use a pair of sync.WaitGroup and
 	// stop channel to shutdown threads in a controlled manner. I decided against
@@ -250,8 +248,7 @@ func taskServe(ctx context.Context, cluster *core.Cluster, args []string, provid
 	}
 
 	// connect to database
-	dbm := must.Return(db.Init())
-	prometheus.MustRegister(sqlstats.NewStatsCollector("limes", dbm.Db))
+	dbm := db.InitORM(must.Return(db.Init()))
 
 	// connect to Hermes RabbitMQ if requested
 	auditor := audittools.NewNullAuditor()
@@ -296,8 +293,7 @@ func taskServeDataMetrics(ctx context.Context, cluster *core.Cluster, args []str
 	}
 
 	// connect to database
-	dbm := must.Return(db.Init())
-	prometheus.MustRegister(sqlstats.NewStatsCollector("limes", dbm.Db))
+	dbm := db.InitORM(must.Return(db.Init()))
 
 	// serve data metrics
 	skipZero := osext.GetenvBool("LIMES_DATA_METRICS_SKIP_ZERO")