Skip to content

Commit

Permalink
Platforms connection in health-check (#325)
Browse files Browse the repository at this point in the history
* Async health check

* Tests adaptation

* Make health configurable externally

* Install health later

* Export ConvertStatus function

* Fix health settings validation

* Attach logger to health

* Address PR comments

* Configuration per indicator and refactoring

* Add health status listener

* Minor tweaks

* Fix indicator interval type

* Fix tests

* Remove unused import

* Extract indicator configuration and address PR comments

* Add error to panic

* Address PR comments

* minor fix

* Add active status to platform

* Platforms connection in health

* Add tests

* Fix formatting

* Hide platform details on unauthorized healthcheck

* Rename migrations

* Fix test last migration

* Add default value to migration

* Last Active default value

* Initial fix health configuration

* fix failing tests

* Address PR comments

* Refactor platform indicator

* Strip indicator error on unauthorized request

* Fix formatting

* Fix squash tag handling
  • Loading branch information
DimitarPetrov authored and dotchev committed Sep 12, 2019
1 parent f30dcba commit 63c53a2
Show file tree
Hide file tree
Showing 17 changed files with 685 additions and 190 deletions.
31 changes: 17 additions & 14 deletions api/healthcheck/healthcheck_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,23 @@
package healthcheck

import (
h "github.com/InVisionApp/go-health"
"context"
gohealth "github.com/InVisionApp/go-health"
"github.com/Peripli/service-manager/pkg/health"
"github.com/Peripli/service-manager/pkg/util"
"net/http"

"github.com/Peripli/service-manager/pkg/log"
"github.com/Peripli/service-manager/pkg/util"
"github.com/Peripli/service-manager/pkg/web"
"net/http"
)

// controller platform controller
type controller struct {
health h.IHealth
health gohealth.IHealth
thresholds map[string]int64
}

// NewController returns a new healthcheck controller with the given health and thresholds
func NewController(health h.IHealth, thresholds map[string]int64) web.Controller {
func NewController(health gohealth.IHealth, thresholds map[string]int64) web.Controller {
return &controller{
health: health,
thresholds: thresholds,
Expand All @@ -46,7 +46,7 @@ func (c *controller) healthCheck(r *web.Request) (*web.Response, error) {
logger := log.C(ctx)
logger.Debugf("Performing health check...")
healthState, _, _ := c.health.State()
healthResult := c.aggregate(healthState)
healthResult := c.aggregate(ctx, healthState)
var status int
if healthResult.Status == health.StatusUp {
status = http.StatusOK
Expand All @@ -56,22 +56,25 @@ func (c *controller) healthCheck(r *web.Request) (*web.Response, error) {
return util.NewJSONResponse(status, healthResult)
}

func (c *controller) aggregate(overallState map[string]h.State) *health.Health {
if len(overallState) == 0 {
func (c *controller) aggregate(ctx context.Context, healthState map[string]gohealth.State) *health.Health {
if len(healthState) == 0 {
return health.New().WithStatus(health.StatusUp)
}

details := make(map[string]interface{})
overallStatus := health.StatusUp
for name, state := range overallState {
for name, state := range healthState {
if state.Fatal && state.ContiguousFailures >= c.thresholds[name] {
overallStatus = health.StatusDown
break
}
}
details := make(map[string]interface{})
for name, state := range overallState {
state.Status = convertStatus(state.Status)
if !web.IsAuthorized(ctx) {
state.Details = nil
state.Err = ""
}
details[name] = state
}

return health.New().WithStatus(overallStatus).WithDetails(details)
}

Expand Down
38 changes: 28 additions & 10 deletions api/healthcheck/healthcheck_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,11 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package healthcheck

import (
"context"
"fmt"
h "github.com/InVisionApp/go-health"
"github.com/Peripli/service-manager/pkg/health"
Expand Down Expand Up @@ -60,14 +62,17 @@ var _ = Describe("Healthcheck controller", func() {
})

Describe("aggregation", func() {
var ctx context.Context
var c *controller
var healths map[string]h.State
var thresholds map[string]int64

BeforeEach(func() {
ctx = context.TODO()
healths = map[string]h.State{
"test1": {Status: "ok"},
"test2": {Status: "ok"},
"test1": {Status: "ok", Details: "details"},
"test2": {Status: "ok", Details: "details"},
"failedState": {Status: "failed", Details: "details", Err: "err"},
}
thresholds = map[string]int64{
"test1": 3,
Expand All @@ -81,7 +86,7 @@ var _ = Describe("Healthcheck controller", func() {

When("No healths are provided", func() {
It("Returns UP", func() {
aggregatedHealth := c.aggregate(nil)
aggregatedHealth := c.aggregate(ctx, nil)
Expect(aggregatedHealth.Status).To(Equal(health.StatusUp))
})
})
Expand All @@ -90,15 +95,15 @@ var _ = Describe("Healthcheck controller", func() {
It("Returns DOWN", func() {
healths["test3"] = h.State{Status: "failed", Fatal: true, ContiguousFailures: 4}
c.thresholds["test3"] = 3
aggregatedHealth := c.aggregate(healths)
aggregatedHealth := c.aggregate(ctx, healths)
Expect(aggregatedHealth.Status).To(Equal(health.StatusDown))
})
})

When("At least one health is DOWN and is not Fatal", func() {
It("Returns UP", func() {
healths["test3"] = h.State{Status: "failed", Fatal: false, ContiguousFailures: 4}
aggregatedHealth := c.aggregate(healths)
aggregatedHealth := c.aggregate(ctx, healths)
Expect(aggregatedHealth.Status).To(Equal(health.StatusUp))
})
})
Expand All @@ -107,21 +112,34 @@ var _ = Describe("Healthcheck controller", func() {
It("Returns UP", func() {
healths["test3"] = h.State{Status: "failed"}
c.thresholds["test3"] = 3
aggregatedHealth := c.aggregate(healths)
aggregatedHealth := c.aggregate(ctx, healths)
Expect(aggregatedHealth.Status).To(Equal(health.StatusUp))
})
})

When("All healths are UP", func() {
It("Returns UP", func() {
aggregatedHealth := c.aggregate(healths)
aggregatedHealth := c.aggregate(ctx, healths)
Expect(aggregatedHealth.Status).To(Equal(health.StatusUp))
})
})

When("Aggregating healths", func() {
It("Includes them as overall details", func() {
aggregatedHealth := c.aggregate(healths)
When("Aggregating health as unauthorized user", func() {
It("should strip details and error", func() {
aggregatedHealth := c.aggregate(ctx, healths)
for name, h := range healths {
h.Status = convertStatus(h.Status)
h.Details = nil
h.Err = ""
Expect(aggregatedHealth.Details[name]).To(Equal(h))
}
})
})

When("Aggregating health as authorized user", func() {
It("should include all details and errors", func() {
ctx = web.ContextWithAuthorization(ctx)
aggregatedHealth := c.aggregate(ctx, healths)
for name, h := range healths {
h.Status = convertStatus(h.Status)
Expect(aggregatedHealth.Details[name]).To(Equal(h))
Expand Down
83 changes: 83 additions & 0 deletions api/healthcheck/platform_indicator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
/*
* Copyright 2018 The Service Manager Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package healthcheck

import (
"context"
"fmt"
"github.com/Peripli/service-manager/pkg/health"
"github.com/Peripli/service-manager/pkg/types"
"github.com/Peripli/service-manager/storage"
)

// NewPlatformIndicator returns new health indicator for platforms of given type
func NewPlatformIndicator(ctx context.Context, repository storage.Repository, fatal func(*types.Platform) bool) health.Indicator {
if fatal == nil {
fatal = func(platform *types.Platform) bool {
return true
}
}
return &platformIndicator{
ctx: ctx,
repository: repository,
fatal: fatal,
}
}

type platformIndicator struct {
repository storage.Repository
ctx context.Context
fatal func(*types.Platform) bool
}

// Name returns the name of the indicator
func (pi *platformIndicator) Name() string {
return health.PlatformsIndicatorName
}

// Status returns status of the health check
func (pi *platformIndicator) Status() (interface{}, error) {
objList, err := pi.repository.List(pi.ctx, types.PlatformType)
if err != nil {
return nil, fmt.Errorf("could not fetch platforms health from storage: %v", err)
}
platforms := objList.(*types.Platforms).Platforms

details := make(map[string]*health.Health)
inactivePlatforms := 0
fatalInactivePlatforms := 0
for _, platform := range platforms {
if platform.Active {
details[platform.Name] = health.New().WithStatus(health.StatusUp).
WithDetail("type", platform.Type)
} else {
details[platform.Name] = health.New().WithStatus(health.StatusDown).
WithDetail("since", platform.LastActive).
WithDetail("type", platform.Type)
inactivePlatforms++
if pi.fatal(platform) {
fatalInactivePlatforms++
}
}
}

if fatalInactivePlatforms > 0 {
err = fmt.Errorf("there are %d inactive platforms %d of them are fatal", inactivePlatforms, fatalInactivePlatforms)
}

return details, err
}
91 changes: 91 additions & 0 deletions api/healthcheck/platform_indicator_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
/*
* Copyright 2018 The Service Manager Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package healthcheck

import (
"context"
"errors"
"github.com/Peripli/service-manager/pkg/health"
"github.com/Peripli/service-manager/pkg/types"
storagefakes2 "github.com/Peripli/service-manager/storage/storagefakes"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"time"
)

var _ = Describe("Platforms Indicator", func() {
var indicator health.Indicator
var repository *storagefakes2.FakeStorage
var ctx context.Context
var platform *types.Platform

BeforeEach(func() {
ctx = context.TODO()
repository = &storagefakes2.FakeStorage{}
platform = &types.Platform{
Name: "test-platform",
Type: "kubernetes",
Active: false,
LastActive: time.Now(),
}
indicator = NewPlatformIndicator(ctx, repository, nil)
})

Context("Name", func() {
It("should not be empty", func() {
Expect(indicator.Name()).Should(Equal(health.PlatformsIndicatorName))
})
})

Context("There are inactive platforms", func() {
BeforeEach(func() {
objectList := &types.Platforms{[]*types.Platform{platform}}
repository.ListReturns(objectList, nil)
})
It("should return error", func() {
details, err := indicator.Status()
health := details.(map[string]*health.Health)[platform.Name]
Expect(err).Should(HaveOccurred())
Expect(health.Details["since"]).ShouldNot(BeNil())
})
})

Context("Storage returns error", func() {
var expectedErr error
BeforeEach(func() {
expectedErr = errors.New("storage err")
repository.ListReturns(nil, expectedErr)
})
It("should return error", func() {
_, err := indicator.Status()
Expect(err).Should(HaveOccurred())
Expect(err.Error()).To(ContainSubstring(expectedErr.Error()))
})
})

Context("All platforms are active", func() {
BeforeEach(func() {
platform.Active = true
objectList := &types.Platforms{[]*types.Platform{platform}}
repository.ListReturns(objectList, nil)
})
It("should not return error", func() {
_, err := indicator.Status()
Expect(err).ShouldNot(HaveOccurred())
})
})
})
Loading

0 comments on commit 63c53a2

Please sign in to comment.