From c0240e8ecdcfaa005e81d02122b159b622181ca9 Mon Sep 17 00:00:00 2001 From: Robbie Caputo Date: Wed, 28 Nov 2018 17:37:39 -0500 Subject: [PATCH 1/7] Add OnComplete hook to Config struct --- health.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/health.go b/health.go index ea97cbc..8a4b238 100644 --- a/health.go +++ b/health.go @@ -83,6 +83,9 @@ type Config struct { // Fatal marks a failing health check so that the // entire health check request fails with a 500 error Fatal bool + + // Hook that gets called when this health check is complete + OnComplete func(state *State) } // State is a struct that contains the results of the latest @@ -268,6 +271,10 @@ func (h *Health) startRunner(cfg *Config, ticker *time.Ticker, stop <-chan struc } h.safeUpdateState(stateEntry) + + if cfg.OnComplete != nil { + cfg.OnComplete(stateEntry) + } } go func() { From 2ba191909632ef6e9418b4fedf89673f61fdec19 Mon Sep 17 00:00:00 2001 From: Robbie Caputo Date: Wed, 28 Nov 2018 17:38:29 -0500 Subject: [PATCH 2/7] Add tests for OnComplete hook --- health_test.go | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/health_test.go b/health_test.go index 2dd610f..9bb4070 100644 --- a/health_test.go +++ b/health_test.go @@ -534,6 +534,44 @@ func TestStartRunner(t *testing.T) { // Since second checker has failed fatally, global healthcheck state should be failed as well Expect(h.Failed()).To(BeTrue()) }) + + t.Run("Should call the onComplete hook when the health check is complete", func(t *testing.T) { + checker := &fakes.FakeICheckable{} + + var calledState *State = nil + called := false + completeFunc := func(state *State) { + called = true + calledState = state + } + + cfgs := []*Config{ + { + Name: "SuperCheck", + Checker: checker, + Interval: testCheckInterval, + Fatal: false, + OnComplete: completeFunc, + }, + } + + h, _, err := setupRunners(cfgs, nil) + + Expect(err).ToNot(HaveOccurred()) + Expect(h).ToNot(BeNil()) + + // Brittle... + time.Sleep(time.Duration(15) * time.Millisecond) + + // Did the ticker fire and create a state entry? + Expect(h.states).To(HaveKey(cfgs[0].Name)) + + // Hook should have been called + Expect(called).To(BeTrue()) + Expect(calledState).ToNot(BeNil()) + Expect(calledState.Name).To(Equal(cfgs[0].Name)) + Expect(calledState.Status).To(Equal("ok")) + }) } func TestStatusListenerOnFail(t *testing.T) { From 5bfa98e9a8cd0cdd3f3f396a1626cc110676aaa8 Mon Sep 17 00:00:00 2001 From: Robbie Caputo Date: Wed, 28 Nov 2018 19:31:40 -0500 Subject: [PATCH 3/7] Put hook call in go routine --- health.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/health.go b/health.go index 8a4b238..f542236 100644 --- a/health.go +++ b/health.go @@ -273,7 +273,7 @@ func (h *Health) startRunner(cfg *Config, ticker *time.Ticker, stop <-chan struc h.safeUpdateState(stateEntry) if cfg.OnComplete != nil { - cfg.OnComplete(stateEntry) + go cfg.OnComplete(stateEntry) } } From 408fd675c09c8a605c9d36fe34f72e6750d4935d Mon Sep 17 00:00:00 2001 From: Robbie Caputo Date: Wed, 28 Nov 2018 19:36:27 -0500 Subject: [PATCH 4/7] Add more tests --- health_test.go | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 48 insertions(+), 2 deletions(-) diff --git a/health_test.go b/health_test.go index 9bb4070..1decbbd 100644 --- a/health_test.go +++ b/health_test.go @@ -535,10 +535,10 @@ func TestStartRunner(t *testing.T) { Expect(h.Failed()).To(BeTrue()) }) - t.Run("Should call the onComplete hook when the health check is complete", func(t *testing.T) { + t.Run("Should call the OnComplete hook when the health check is complete", func(t *testing.T) { checker := &fakes.FakeICheckable{} - var calledState *State = nil + var calledState *State called := false completeFunc := func(state *State) { called = true @@ -572,6 +572,52 @@ func TestStartRunner(t *testing.T) { Expect(calledState.Name).To(Equal(cfgs[0].Name)) Expect(calledState.Status).To(Equal("ok")) }) + + t.Run("Modifying the state in the OnComplete hook should not modify the one saved in the states map", func(t *testing.T) { + checker := &fakes.FakeICheckable{} + + var calledState *State + called := false + changedName := "Guybrush Threepwood" + changedStatus := "never" + completeFunc := func(state *State) { + called = true + state.Name = changedName + state.Status = changedStatus + calledState = state + } + + cfgs := []*Config{ + { + Name: "CheckIt", + Checker: checker, + Interval: testCheckInterval, + Fatal: false, + OnComplete: completeFunc, + }, + } + + h, _, err := setupRunners(cfgs, nil) + + Expect(err).ToNot(HaveOccurred()) + Expect(h).ToNot(BeNil()) + + // Brittle... + time.Sleep(time.Duration(15) * time.Millisecond) + + // Did the ticker fire and create a state entry? + Expect(h.states).To(HaveKey(cfgs[0].Name)) + + // Hook should have been called + Expect(called).To(BeTrue()) + Expect(calledState).ToNot(BeNil()) + + //changed status in OnComplete should not affect internal states map + Expect(calledState.Name).To(Equal(changedName)) + Expect(calledState.Status).To(Equal(changedStatus)) + Expect(h.states[cfgs[0].Name].Name).To(Equal(cfgs[0].Name)) + Expect(h.states[cfgs[0].Name].Status).To(Equal("ok")) + }) } func TestStatusListenerOnFail(t *testing.T) { From 54cc7200ee15418fcf38e18692910aa4107cc22f Mon Sep 17 00:00:00 2001 From: Robbie Caputo Date: Mon, 3 Dec 2018 17:59:44 -0500 Subject: [PATCH 5/7] Update README and add example for OnComplete hook --- README.md | 32 +++++++++++++++------------ examples/on-complete-hook/README.md | 34 +++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 14 deletions(-) create mode 100644 examples/on-complete-hook/README.md diff --git a/README.md b/README.md index c35e29f..bbd8efd 100644 --- a/README.md +++ b/README.md @@ -7,15 +7,15 @@ # go-health -A library that enables *async* dependency health checking for services running on an orchastrated container platform such as kubernetes or mesos. +A library that enables *async* dependency health checking for services running on an orchestrated container platform such as kubernetes or mesos. ## Why is this important? -Container orchestration platforms require that the underlying service(s) expose a "healthcheck" which is used by the platform to determine whether the container is in a good or bad state. +Container orchestration platforms require that the underlying service(s) expose a "health check" which is used by the platform to determine whether the container is in a good or bad state. -While this can be achieved by simply exposing a `/status` endpoint that perfoms synchronous checks against its dependencies (followed by returning a `200` or `non-200` status code), it is not optimal for a number of reasons: +While this can be achieved by simply exposing a `/status` endpoint that performs synchronous checks against its dependencies (followed by returning a `200` or `non-200` status code), it is not optimal for a number of reasons: * **It does not scale** - + The more dependencies you add, the longer your healthcheck will take to complete (and potentially cause your service to be killed off by the orchestration platform). + + The more dependencies you add, the longer your health check will take to complete (and potentially cause your service to be killed off by the orchestration platform). + Depending on the complexity of a given dependency, your check may be fairly involved where it is _okay_ for it to take `30s+` to complete. * **It adds unnecessary load on yours deps or at worst, becomes a DoS target** + **Non-malicious scenario** @@ -26,10 +26,10 @@ While this can be achieved by simply exposing a `/status` endpoint that perfoms With that said, not everyone _needs_ asynchronous checks. If your service has one dependency (and that is unlikely to change), it is trivial to write a basic, synchronous check and it will probably suffice. -However, if you anticipate that your service will have several dependencies, with varying degrees of complexity for determing their health state - you should probably think about introducing asynchronous health checks. +However, if you anticipate that your service will have several dependencies, with varying degrees of complexity for determining their health state - you should probably think about introducing asynchronous health checks. ## How does this library help? -Writing an async healthchecking framework for your service is not a trivial task, especially if Go is not your primary language. +Writing an async health checking framework for your service is not a trivial task, especially if Go is not your primary language. This library: @@ -37,12 +37,10 @@ This library: * Allows you to define warning and fatal thresholds. * Will run your dependency checks on a given interval, in the background. **[1]** * Exposes a way for you to gather the check results in a *fast* and *thread-safe* manner to help determine the final status of your `/status` endpoint. **[2]** -* Comes bundled w/ [pre-built checkers](/checkers) for well-known dependencies such as `Redis`, `HTTP`. +* Comes bundled w/ [pre-built checkers](/checkers) for well-known dependencies such as `Redis`, `Mongo`, `HTTP` and more. * Makes it simple to implement and provide your own checkers (by adhering to the checker interface). -* Is test-friendly - + Provides an easy way to disable dependency health checking. - + Uses an interface for its dependencies, allowing you to insert fakes/mocks at test time. -* Allows you to trigger listener functions when a health check fails or recovers. **[3]** +* Allows you to trigger listener functions when your health checks fail or recover using the `IStatusListener` interface. +* Allows you to run custom logic when a specific health check completes by using the `OnComplete` hook. **[1]** Make sure to run your checks on a "sane" interval - ie. if you are checking your Redis dependency once every five minutes, your service is essentially running _blind_ @@ -54,8 +52,6 @@ you to query that data via `.State()`. Alternatively, you can use one of the pre-built HTTP handlers for your `/healthcheck` endpoint (and thus not have to manually inspect the state data). -**[3]** By utilizing an implementation of the `IStatusListener` interface - ## Example For _full_ examples, look through the [examples dir](examples/) @@ -92,7 +88,7 @@ h.AddChecks([]*health.Config{ }) ``` -3. Start the healthcheck +3. Start the health check ```golang h.Start() @@ -126,7 +122,15 @@ output would look something like this: ## Additional Documentation * [Examples](/examples) * [Status Listeners](/examples/status-listener) + * [OnComplete Hook](/examples/on-complete-hook) * [Checkers](/checkers) +## OnComplete Hook VS IStatusListener +At first glance it may seem that these two features provide the same functionality. However, they are meant for two different use cases: + +The `IStatusListener` is useful when you want to run a custom function in the event that the overall status of your health checks change. I.E. if `go-health` is currently checking the health for two different dependencies A and B, you may want to trip a circuit breaker for A and/or B. You could also put your service in a state where it will notify callers that it is not currently operating correctly. The opposite can be done when your service recovers. + +The `OnComplete` hook is called whenever a health check for an individual dependency is complete. This means that the function you register with the hook gets called every single time `go-health` completes the check. It's completely possible to register different functions with each configured health check or not to hook into the completion of certain health checks entirely. For instance, this can be useful if you want to perform cleanup after a complex health check or if you want to send metrics to your APM software when a health check completes. It is important to keep in mind that this hook effectively gets called on roughly the same interval you define for the health check. + ## Contributing All PR's are welcome, as long as they are well tested. Follow the typical fork->branch->pr flow. diff --git a/examples/on-complete-hook/README.md b/examples/on-complete-hook/README.md new file mode 100644 index 0000000..24cee5e --- /dev/null +++ b/examples/on-complete-hook/README.md @@ -0,0 +1,34 @@ +# OnComplete Hook + +A function can be registered with the `OnComplete` hook by providing a func to the `OnComplete` field of the `health.Config` struct when setting up your health check. The function should have the following signature: `func (state *health.State)`. +The field can be left out entirely or set to `nil` if you do not wish to register a function with the hook. + +Here is a simple example of configuring a `OnComplete` hook for a health check: + +```golang +func main() { + // Create a new health instance + h := health.New() + + // Instantiate your custom check + cc := &customCheck{} + + // Add the checks to the health instance + h.AddChecks([]*health.Config{ + { + Name: "good-check", + Checker: cc, + Interval: time.Duration(2) * time.Second, + Fatal: true, + OnComplete: MyHookFunc + }, + }) + + ... +} + +func MyHookFunc(state *health.State) { + log.Println("The state of %s is %s", state.Name, state.Status) + // Other custom logic here... +} +``` \ No newline at end of file From 9183988f01e968e932ed658d0c0fccaa7982bcd0 Mon Sep 17 00:00:00 2001 From: Robbie Caputo Date: Mon, 3 Dec 2018 18:00:29 -0500 Subject: [PATCH 6/7] Update checkers README --- checkers/README.md | 323 +++++++++++++++++++++++---------------------- 1 file changed, 164 insertions(+), 159 deletions(-) diff --git a/checkers/README.md b/checkers/README.md index f010cd6..ec8338c 100644 --- a/checkers/README.md +++ b/checkers/README.md @@ -43,168 +43,173 @@ The SQL DB checker has implementations for the following interfaces: - `SQLQueryer`, which encloses `QueryContext` in [`sql.DB`](https://golang.org/pkg/database/sql/#DB.QueryContext), [`sql.Conn`](https://golang.org/pkg/database/sql/#Conn.QueryContext), [`sql.Stmt`](https://golang.org/pkg/database/sql/#Stmt.QueryContext), and [`sql.Tx`](https://golang.org/pkg/database/sql/#Tx.QueryContext) - `SQLExecer`, which encloses `ExecContext` in [`sql.DB`](https://golang.org/pkg/database/sql/#DB.ExecContext), [`sql.Conn`](https://golang.org/pkg/database/sql/#Conn.ExecContext), [`sql.Stmt`](https://golang.org/pkg/database/sql/#Stmt.ExecContext), and [`sql.Tx`](https://golang.org/pkg/database/sql/#Tx.ExecContext) - #### SQLConfig - The `SQLConfig` struct is required when using the SQL DB health check. It **must** contain an inplementation of one of either `SQLPinger`, `SQLQueryer`, or `SQLExecer`. - - If `SQLQueryer` or `SQLExecer` are implemented, then `Query` must be valid (len > 0). - - Additionally, if `SQLQueryer` or `SQLExecer` are implemented, you have the option to also set either the `QueryerResultHandler` or `ExecerResultHandler` functions. These functions allow you to evaluate the result of a query or exec operation. If you choose not to implement these yourself, the default handlers are used. - - The default `ExecerResultHandler` is successful if the passed exec operation affected one and only one row. - - The default `QueryerResultHandler` is successful if the passed query operation returned one and only one row. - - #### SQLPinger - - Use the `SQLPinger` interface if your health check is only concerned with your application's database connectivity. All you need to do is set the `Pinger` value in your `SQLConfig`. - - ```golang - db, err := sql.Open("mysql", dsn) - if err != nil { - return err - } - - sqlCheck, err := checkers.NewSQL(&checkers.SQLConfig{ - Pinger: db - }) - if err != nil { - return err - } - - hc := health.New() - healthCheck.AddCheck(&health.Config{ - Name: "sql-check", - Checker: sqlCheck, - Interval: time.Duration(3) * time.Second, - Fatal: true, - }) - ``` - - #### SQLQueryer - Use the `SQLQueryer` interface if your health check requires you to read rows from your database. You can optionally supply a query result handler function. If you don't supply one, the default function will be used. The function signature for the handler is: - - ```golang - type SQLQueryerResultHandler func(rows *sql.Rows) (bool, error) - ``` - The default query handler returns true if there was exactly one row in the resultset: - - ```golang - func DefaultQueryHandler(rows *sql.Rows) (bool, error) { - defer rows.Close() - - numRows := 0 - for rows.Next() { - numRows++ - } - - return numRows == 1, nil - } - ``` - **IMPORTANT**: Note that your query handler is responsible for closing the passed `*sql.Rows` value. - - Sample `SQLQueryer` implementation: - - ```golang - // this is our custom query row handler - func myQueryHandler(rows *sql.Rows) (bool, error) { - defer rows.Close() - - var healthValue string - for rows.Next() { - // this query will ever return at most one row - if err := rows.Scan(&healthValue); err != nil { - return false, err - } - } - - return healthValue == "ok", nil - } - - db, err := sql.Open("mysql", dsn) - if err != nil { - return err - } - - // we pass the id we are looking for inside the params value - sqlCheck, err := checkers.NewSQL(&checkers.SQLConfig{ - Queryerer: db, - Query: "SELECT healthValue FROM some_table WHERE id = ?", - Params: []interface{}{1}, - QueryerResultHandler: myQueryHandler - }) - if err != nil { - return err - } - - hc := health.New() - healthCheck.AddCheck(&health.Config{ - Name: "sql-check", - Checker: sqlCheck, - Interval: time.Duration(3) * time.Second, - Fatal: true, - }) - ``` - - #### SQLExecer - Use the `SQLExecer` interface if your health check requires you to update or insert to your database. You can optionally supply an exec result handler function. If you don't supply one, the default function will be used. The function signature for the handler is: - - ```golang - type SQLExecerResultHandler func(result sql.Result) (bool, error) - ``` - The default exec handler returns true if there was exactly one affected row: - - ```golang - func DefaultExecHandler(result sql.Result) (bool, error) { - affectedRows, err := result.RowsAffected() - if err != nil { - return false, err - } - - return affectedRows == int64(1), nil - } - ``` - - Sample `SQLExecer ` implementation: - - ```golang - // this is our custom exec result handler - func myExecHandler(result sql.Result) (bool, error) { - insertId, err := result.LastInsertId() - if err != nil { - return false, err - } - - // for this example, a check isn't valid - // until after the 100th iteration - return insertId > int64(100), nil - } - - db, err := sql.Open("mysql", dsn) - if err != nil { - return err - } - - sqlCheck, err := checkers.NewSQL(&checkers.SQLConfig{ - Execer: db, - Query: "INSERT INTO checks (checkTS) VALUES (NOW())", - ExecerResultHandler: myExecHandler - }) - if err != nil { - return err - } - - hc := health.New() - healthCheck.AddCheck(&health.Config{ - Name: "sql-check", - Checker: sqlCheck, - Interval: time.Duration(3) * time.Second, - Fatal: true, - }) - ``` +#### SQLConfig +The `SQLConfig` struct is required when using the SQL DB health check. It **must** contain an inplementation of one of either `SQLPinger`, `SQLQueryer`, or `SQLExecer`. + +If `SQLQueryer` or `SQLExecer` are implemented, then `Query` must be valid (len > 0). + +Additionally, if `SQLQueryer` or `SQLExecer` are implemented, you have the option to also set either the `QueryerResultHandler` or `ExecerResultHandler` functions. These functions allow you to evaluate the result of a query or exec operation. If you choose not to implement these yourself, the default handlers are used. + +The default `ExecerResultHandler` is successful if the passed exec operation affected one and only one row. + +The default `QueryerResultHandler` is successful if the passed query operation returned one and only one row. + +#### SQLPinger +Use the `SQLPinger` interface if your health check is only concerned with your application's database connectivity. All you need to do is set the `Pinger` value in your `SQLConfig`. + +```golang + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + + sqlCheck, err := checkers.NewSQL(&checkers.SQLConfig{ + Pinger: db + }) + if err != nil { + return err + } + + hc := health.New() + healthCheck.AddCheck(&health.Config{ + Name: "sql-check", + Checker: sqlCheck, + Interval: time.Duration(3) * time.Second, + Fatal: true, + }) +``` + +#### SQLQueryer +Use the `SQLQueryer` interface if your health check requires you to read rows from your database. You can optionally supply a query result handler function. If you don't supply one, the default function will be used. The function signature for the handler is: + +```golang +type SQLQueryerResultHandler func(rows *sql.Rows) (bool, error) +``` +The default query handler returns true if there was exactly one row in the resultset: + +```golang + func DefaultQueryHandler(rows *sql.Rows) (bool, error) { + defer rows.Close() + + numRows := 0 + for rows.Next() { + numRows++ + } + + return numRows == 1, nil + } +``` + +**IMPORTANT**: Note that your query handler is responsible for closing the passed `*sql.Rows` value. + +Sample `SQLQueryer` implementation: + +```golang + // this is our custom query row handler + func myQueryHandler(rows *sql.Rows) (bool, error) { + defer rows.Close() + + var healthValue string + for rows.Next() { + // this query will ever return at most one row + if err := rows.Scan(&healthValue); err != nil { + return false, err + } + } + + return healthValue == "ok", nil + } + + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + + // we pass the id we are looking for inside the params value + sqlCheck, err := checkers.NewSQL(&checkers.SQLConfig{ + Queryerer: db, + Query: "SELECT healthValue FROM some_table WHERE id = ?", + Params: []interface{}{1}, + QueryerResultHandler: myQueryHandler + }) + if err != nil { + return err + } + + hc := health.New() + healthCheck.AddCheck(&health.Config{ + Name: "sql-check", + Checker: sqlCheck, + Interval: time.Duration(3) * time.Second, + Fatal: true, + }) +``` + +#### SQLExecer +Use the `SQLExecer` interface if your health check requires you to update or insert to your database. You can optionally supply an exec result handler function. If you don't supply one, the default function will be used. The function signature for the handler is: + +```golang +type SQLExecerResultHandler func(result sql.Result) (bool, error) +``` + +The default exec handler returns true if there was exactly one affected row: + +```golang + func DefaultExecHandler(result sql.Result) (bool, error) { + affectedRows, err := result.RowsAffected() + if err != nil { + return false, err + } + + return affectedRows == int64(1), nil + } +``` + +Sample `SQLExecer ` implementation: + +```golang + // this is our custom exec result handler + func myExecHandler(result sql.Result) (bool, error) { + insertId, err := result.LastInsertId() + if err != nil { + return false, err + } + + // for this example, a check isn't valid + // until after the 100th iteration + return insertId > int64(100), nil + } + + db, err := sql.Open("mysql", dsn) + if err != nil { + return err + } + + sqlCheck, err := checkers.NewSQL(&checkers.SQLConfig{ + Execer: db, + Query: "INSERT INTO checks (checkTS) VALUES (NOW())", + ExecerResultHandler: myExecHandler + }) + if err != nil { + return err + } + + hc := health.New() + healthCheck.AddCheck(&health.Config{ + Name: "sql-check", + Checker: sqlCheck, + Interval: time.Duration(3) * time.Second, + Fatal: true, + }) +``` ### Mongo -Planned, but PR's welcome! +Mongo checker allows you to test if an instance of MongoDB is available by using the underlying driver's ping method or check whether a collection exists or not. + +To make use of it, initialize a `MongoConfig` struct and pass it to `checkers.NewMongo(...)`. + +The `MongoConfig` struct must specify either one or both of the `Ping` or `Collection` fields. ### Reachable From 637c97505ba15ac6c7101e5f8b42e3eed40c3ace Mon Sep 17 00:00:00 2001 From: Robbie Caputo Date: Mon, 3 Dec 2018 20:23:34 -0500 Subject: [PATCH 7/7] Add more links to README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index bbd8efd..b54a870 100644 --- a/README.md +++ b/README.md @@ -39,8 +39,8 @@ This library: * Exposes a way for you to gather the check results in a *fast* and *thread-safe* manner to help determine the final status of your `/status` endpoint. **[2]** * Comes bundled w/ [pre-built checkers](/checkers) for well-known dependencies such as `Redis`, `Mongo`, `HTTP` and more. * Makes it simple to implement and provide your own checkers (by adhering to the checker interface). -* Allows you to trigger listener functions when your health checks fail or recover using the `IStatusListener` interface. -* Allows you to run custom logic when a specific health check completes by using the `OnComplete` hook. +* Allows you to trigger listener functions when your health checks fail or recover using the [`IStatusListener` interface](#oncomplete-hook-vs-istatuslistener). +* Allows you to run custom logic when a specific health check completes by using the [`OnComplete` hook](#oncomplete-hook-vs-istatuslistener). **[1]** Make sure to run your checks on a "sane" interval - ie. if you are checking your Redis dependency once every five minutes, your service is essentially running _blind_