diff --git a/README.md b/README.md index 93630f50..5e273832 100644 --- a/README.md +++ b/README.md @@ -212,6 +212,19 @@ environment. LOGLEVEL=info ./sql_exporter ``` +Database specific configurations +-------------------------------- + +For some database backends some special functionality is available : + +* cloudsql-postgres: a special `*` caracter can be used to query all databases + accessible by the account +* cloudsql-mysql : same as above +* rds-postgres : this type of URL expects a working AWS configuration + which will use action the equivalent of `rds generate-db-auth-token` + for the password. For this driver, the `AWS_REGION` environment variable + must be set. + Why this exporter exists ======================== diff --git a/config.yml.dist b/config.yml.dist index e5fbd5bf..3fc50173 100644 --- a/config.yml.dist +++ b/config.yml.dist @@ -178,3 +178,18 @@ jobs: node_name, schema_name, projection_name; +- name: "rds" + interval: '5m' + connections: + - 'rds-postgres://postgres_usr:AUTHTOKEN@mypostgresql.c6c8mwvfdgv0.us-west-2.rds.amazonaws.com/db_name' + queries: + - name: "running_queries" + help: "Number of running queries" + labels: + - "datname" + - "usename" + values: + - "count" + query: | + SELECT datname::text, usename::text, COUNT(*)::float AS count + FROM pg_stat_activity GROUP BY datname, usename; diff --git a/job.go b/job.go index e3f9ec8e..b737a261 100644 --- a/job.go +++ b/job.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "net/url" + "os" "regexp" "strconv" "strings" @@ -23,6 +24,9 @@ import ( "github.com/snowflakedb/gosnowflake" _ "github.com/vertica/vertica-sql-go" // register the Vertica driver sqladmin "google.golang.org/api/sqladmin/v1beta4" + + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/rds/rdsutils" ) var ( @@ -91,7 +95,6 @@ func (j *Job) updateConnections() { // parse the connection URLs and create a connection object for each if len(j.conns) < len(j.Connections) { for _, conn := range j.Connections { - // Check if we need to use cloudsql driver if useCloudSQL, cloudsqlDriver := isValidCloudSQLDriver(conn); useCloudSQL { // Do CloudSQL stuff @@ -221,6 +224,27 @@ func (j *Job) updateConnections() { }) continue } + if strings.HasPrefix(conn, "rds-postgres://") { + // reuse postgres SQLDriver by stripping rds- from connexion URL after building the RDS + // authentication token + conn = strings.TrimPrefix(conn, "rds-") + // FIXME - parsing twice the conn url to extract host & username + u, err := url.Parse(conn) + if err != nil { + level.Error(j.log).Log("msg", "Failed to parse URL", "url", conn, "err", err) + continue + } + region := os.Getenv("AWS_REGION") + sess := session.Must(session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + })) + token, err := rdsutils.BuildAuthToken(u.Host, region, u.User.Username(), sess.Config.Credentials) + if err != nil { + level.Error(j.log).Log("msg", "Failed to parse URL", "url", conn, "err", err) + continue + } + conn = strings.Replace(conn, "AUTHTOKEN", url.QueryEscape(token), 1) + } u, err := url.Parse(conn) if err != nil { diff --git a/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/builder.go b/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/builder.go new file mode 100644 index 00000000..552acd13 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/builder.go @@ -0,0 +1,127 @@ +package rdsutils + +import ( + "fmt" + "net/url" + + "github.com/aws/aws-sdk-go/aws/awserr" + "github.com/aws/aws-sdk-go/aws/credentials" +) + +// ConnectionFormat is the type of connection that will be +// used to connect to the database +type ConnectionFormat string + +// ConnectionFormat enums +const ( + NoConnectionFormat ConnectionFormat = "" + TCPFormat ConnectionFormat = "tcp" +) + +// ErrNoConnectionFormat will be returned during build if no format had been +// specified +var ErrNoConnectionFormat = awserr.New("NoConnectionFormat", "No connection format was specified", nil) + +// ConnectionStringBuilder is a builder that will construct a connection +// string with the provided parameters. params field is required to have +// a tls specification and allowCleartextPasswords must be set to true. +type ConnectionStringBuilder struct { + dbName string + endpoint string + region string + user string + creds *credentials.Credentials + + connectFormat ConnectionFormat + params url.Values +} + +// NewConnectionStringBuilder will return an ConnectionStringBuilder +func NewConnectionStringBuilder(endpoint, region, dbUser, dbName string, creds *credentials.Credentials) ConnectionStringBuilder { + return ConnectionStringBuilder{ + dbName: dbName, + endpoint: endpoint, + region: region, + user: dbUser, + creds: creds, + } +} + +// WithEndpoint will return a builder with the given endpoint +func (b ConnectionStringBuilder) WithEndpoint(endpoint string) ConnectionStringBuilder { + b.endpoint = endpoint + return b +} + +// WithRegion will return a builder with the given region +func (b ConnectionStringBuilder) WithRegion(region string) ConnectionStringBuilder { + b.region = region + return b +} + +// WithUser will return a builder with the given user +func (b ConnectionStringBuilder) WithUser(user string) ConnectionStringBuilder { + b.user = user + return b +} + +// WithDBName will return a builder with the given database name +func (b ConnectionStringBuilder) WithDBName(dbName string) ConnectionStringBuilder { + b.dbName = dbName + return b +} + +// WithParams will return a builder with the given params. The parameters +// will be included in the connection query string +// +// Example: +// v := url.Values{} +// v.Add("tls", "rds") +// b := rdsutils.NewConnectionBuilder(endpoint, region, user, dbname, creds) +// connectStr, err := b.WithParams(v).WithTCPFormat().Build() +func (b ConnectionStringBuilder) WithParams(params url.Values) ConnectionStringBuilder { + b.params = params + return b +} + +// WithFormat will return a builder with the given connection format +func (b ConnectionStringBuilder) WithFormat(f ConnectionFormat) ConnectionStringBuilder { + b.connectFormat = f + return b +} + +// WithTCPFormat will set the format to TCP and return the modified builder +func (b ConnectionStringBuilder) WithTCPFormat() ConnectionStringBuilder { + return b.WithFormat(TCPFormat) +} + +// Build will return a new connection string that can be used to open a connection +// to the desired database. +// +// Example: +// b := rdsutils.NewConnectionStringBuilder(endpoint, region, user, dbname, creds) +// connectStr, err := b.WithTCPFormat().Build() +// if err != nil { +// panic(err) +// } +// const dbType = "mysql" +// db, err := sql.Open(dbType, connectStr) +func (b ConnectionStringBuilder) Build() (string, error) { + if b.connectFormat == NoConnectionFormat { + return "", ErrNoConnectionFormat + } + + authToken, err := BuildAuthToken(b.endpoint, b.region, b.user, b.creds) + if err != nil { + return "", err + } + + connectionStr := fmt.Sprintf("%s:%s@%s(%s)/%s", + b.user, authToken, string(b.connectFormat), b.endpoint, b.dbName, + ) + + if len(b.params) > 0 { + connectionStr = fmt.Sprintf("%s?%s", connectionStr, b.params.Encode()) + } + return connectionStr, nil +} diff --git a/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/connect.go b/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/connect.go new file mode 100644 index 00000000..0cf078ae --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/connect.go @@ -0,0 +1,67 @@ +package rdsutils + +import ( + "net/http" + "strings" + "time" + + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/signer/v4" +) + +// BuildAuthToken will return an authorization token used as the password for a DB +// connection. +// +// * endpoint - Endpoint consists of the port needed to connect to the DB. : +// * region - Region is the location of where the DB is +// * dbUser - User account within the database to sign in with +// * creds - Credentials to be signed with +// +// The following example shows how to use BuildAuthToken to create an authentication +// token for connecting to a MySQL database in RDS. +// +// authToken, err := BuildAuthToken(dbEndpoint, awsRegion, dbUser, awsCreds) +// +// // Create the MySQL DNS string for the DB connection +// // user:password@protocol(endpoint)/dbname? +// connectStr = fmt.Sprintf("%s:%s@tcp(%s)/%s?allowCleartextPasswords=true&tls=rds", +// dbUser, authToken, dbEndpoint, dbName, +// ) +// +// // Use db to perform SQL operations on database +// db, err := sql.Open("mysql", connectStr) +// +// See http://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html +// for more information on using IAM database authentication with RDS. +func BuildAuthToken(endpoint, region, dbUser string, creds *credentials.Credentials) (string, error) { + // the scheme is arbitrary and is only needed because validation of the URL requires one. + if !(strings.HasPrefix(endpoint, "http://") || strings.HasPrefix(endpoint, "https://")) { + endpoint = "https://" + endpoint + } + + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return "", err + } + values := req.URL.Query() + values.Set("Action", "connect") + values.Set("DBUser", dbUser) + req.URL.RawQuery = values.Encode() + + signer := v4.Signer{ + Credentials: creds, + } + _, err = signer.Presign(req, nil, "rds-db", region, 15*time.Minute, time.Now()) + if err != nil { + return "", err + } + + url := req.URL.String() + if strings.HasPrefix(url, "http://") { + url = url[len("http://"):] + } else if strings.HasPrefix(url, "https://") { + url = url[len("https://"):] + } + + return url, nil +} diff --git a/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/doc.go b/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/doc.go new file mode 100644 index 00000000..9f3e8813 --- /dev/null +++ b/vendor/github.com/aws/aws-sdk-go/service/rds/rdsutils/doc.go @@ -0,0 +1,20 @@ +// Package rdsutils is used to generate authentication tokens used to +// connect to a givent Amazon Relational Database Service (RDS) database. +// +// Before using the authentication please visit the docs here to ensure +// the database has the proper policies to allow for IAM token authentication. +// https://docs.aws.amazon.com/AmazonRDS/latest/UserGuide/UsingWithRDS.IAMDBAuth.html#UsingWithRDS.IAMDBAuth.Availability +// +// When building the connection string, there are two required parameters that are needed to be set on the query. +// +// - tls +// +// - allowCleartextPasswords must be set to true +// +// Example creating a basic auth token with the builder: +// v := url.Values{} +// v.Add("tls", "tls_profile_name") +// v.Add("allowCleartextPasswords", "true") +// b := rdsutils.NewConnectionStringBuilder(endpoint, region, user, dbname, creds) +// connectStr, err := b.WithTCPFormat().WithParams(v).Build() +package rdsutils diff --git a/vendor/modules.txt b/vendor/modules.txt index 9c748023..ef8a5fb0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -152,6 +152,7 @@ github.com/aws/aws-sdk-go/private/protocol/restjson github.com/aws/aws-sdk-go/private/protocol/xml/xmlutil github.com/aws/aws-sdk-go/service/athena github.com/aws/aws-sdk-go/service/athena/athenaiface +github.com/aws/aws-sdk-go/service/rds/rdsutils github.com/aws/aws-sdk-go/service/sso github.com/aws/aws-sdk-go/service/sso/ssoiface github.com/aws/aws-sdk-go/service/ssooidc