feat: validate release status during cluster prepare cmd (#300)

* chore: move AppRelease struct to pkg/types package

* feat: check for release readiness

* chore: fix build

* feat: add an exit timeout of 10* no of charts sec

* chore: rename func

cli/cmd/cluster_prepare.go

@@ -177,16 +177,11 @@ func (r *runners) prepareCluster(_ *cobra.Command, args []string) error {
 		return errors.Wrap(err, "get cluster kubeconfig")
 	}
 
-	// we need to have a status on the release so that we can wait for it
-	// to be pushed to the oci registry. this is a terrible hack working
-	// around that part. a pr is in progress to deliver this status
-
-	if len(release.Charts) == 0 {
-		return errors.New("no charts found in release")
+	isReleaseReady, err := isReleaseReadyToInstall(r, log, *release)
+	if err != nil || !isReleaseReady {
+		return errors.Wrap(err, "release not ready")
 	}
 
-	time.Sleep(time.Second * 30)
-
 	// run preflights
 
 	// install the chart or application
@@ -221,7 +216,7 @@ func installChartRelease(appSlug string, releaseSequence int64, chartName string
 
 	configJSON := fmt.Sprintf(`{"auths":{"%s":%s}}`, registryHostname, encodedAuthConfigJSON)
 
-	credentialsFile, err := ioutil.TempFile("", "credentials")
+	credentialsFile, err := os.CreateTemp("", "credentials")
 	if err != nil {
 		return "", errors.Wrap(err, "failed to create credentials file")
 	}
@@ -234,12 +229,12 @@ func installChartRelease(appSlug string, releaseSequence int64, chartName string
 
 	settings := cli.New()
 
-	kubeconfigFile, err := ioutil.TempFile("", "kubeconfig")
+	kubeconfigFile, err := os.CreateTemp("", "kubeconfig")
 	if err != nil {
 		return "", errors.Wrap(err, "failed to create kubeconfig file")
 	}
 	defer os.RemoveAll(kubeconfigFile.Name())
-	if err := ioutil.WriteFile(kubeconfigFile.Name(), kubeconfig, 0644); err != nil {
+	if err := os.WriteFile(kubeconfigFile.Name(), kubeconfig, 0644); err != nil {
 		return "", errors.Wrap(err, "failed to write kubeconfig file")
 	}
 	settings.KubeConfig = kubeconfigFile.Name()
@@ -367,3 +362,56 @@ func prepareRelease(r *runners, log *logger.Logger) (*types.ReleaseInfo, error)
 
 	return release, nil
 }
+
+func isReleaseReadyToInstall(r *runners, log *logger.Logger, release types.ReleaseInfo) (bool, error) {
+	if len(release.Charts) == 0 {
+		return false, errors.New("no charts found in release")
+	}
+
+	timeout := time.Duration(10*len(release.Charts)) * time.Second
+	timer := time.NewTimer(timeout)
+	defer timer.Stop()
+
+	for {
+		select {
+		case <-timer.C:
+			return false, errors.Errorf("timed out waiting for release to be ready after %s", timeout)
+		default:
+			appRelease, err := r.api.GetRelease(r.appID, r.appType, release.Sequence)
+			if err != nil {
+				return false, errors.Wrap(err, "failed to get release")
+			}
+
+			ready, err := areReleaseChartsPushed(appRelease.Charts)
+			if err != nil {
+				return false, errors.Wrap(err, "failed to check release charts")
+			} else if ready {
+				return true, nil
+			}
+
+			time.Sleep(time.Second * 2)
+		}
+	}
+}
+
+func areReleaseChartsPushed(charts []types.Chart) (bool, error) {
+	if len(charts) == 0 {
+		return false, errors.New("no charts found in release")
+	}
+
+	pushedChartsCount := 0
+	for _, chart := range charts {
+		switch chart.Status {
+		case types.ChartStatusPushed:
+			pushedChartsCount++
+		case types.ChartStatusUnknown, types.ChartStatusPushing:
+			// wait for the chart to be pushed
+		case types.ChartStatusError:
+			return false, errors.Errorf("chart %q failed to push: %s", chart.Name, chart.Error)
+		default:
+			return false, errors.Errorf("unknown release chart status %q", chart.Status)
+		}
+	}
+
+	return pushedChartsCount == len(charts), nil
+}

cli/cmd/cluster_prepare_test.go

@@ -0,0 +1,48 @@
+package cmd
+
+import (
+	"testing"
+
+	"github.com/replicatedhq/replicated/pkg/types"
+)
+
+func Test_areReleaseChartsReady(t *testing.T) {
+	tests := []struct {
+		name    string
+		charts  []types.Chart
+		want    bool
+		wantErr bool
+	}{
+		{"nil charts", nil, false, true},
+		{"no charts", []types.Chart{}, false, true},
+		{"one chart, no status", []types.Chart{{}}, false, true},
+		{"one chart, status unkown", []types.Chart{{Status: types.ChartStatusUnknown}}, false, false},
+		{"one chart, status pushing", []types.Chart{{Status: types.ChartStatusPushing}}, false, false},
+		{"one chart, status pushed", []types.Chart{{Status: types.ChartStatusPushed}}, true, false},
+		{"one chart, status error", []types.Chart{{Status: types.ChartStatusError}}, false, true},
+		{"two charts, status pushed", []types.Chart{{Status: types.ChartStatusPushed}, {Status: types.ChartStatusPushed}}, true, false},
+		{"two charts, status pushed and pushing", []types.Chart{{Status: types.ChartStatusPushed}, {Status: types.ChartStatusPushing}}, false, false},
+		{"two charts, status pushed and error", []types.Chart{{Status: types.ChartStatusPushed}, {Status: types.ChartStatusError}}, false, true},
+		{"two charts, status pushed and unknown", []types.Chart{{Status: types.ChartStatusPushed}, {Status: types.ChartStatusUnknown}}, false, false},
+		{"two charts, status pushing and error", []types.Chart{{Status: types.ChartStatusPushing}, {Status: types.ChartStatusError}}, false, true},
+		{"two charts, status pushing and unknown", []types.Chart{{Status: types.ChartStatusPushing}, {Status: types.ChartStatusUnknown}}, false, false},
+		{"two charts, status error and unknown", []types.Chart{{Status: types.ChartStatusError}, {Status: types.ChartStatusUnknown}}, false, true},
+		{"two charts, status error and error", []types.Chart{{Status: types.ChartStatusError}, {Status: types.ChartStatusError}}, false, true},
+		{"three charts, status pushed, pushing, and error", []types.Chart{{Status: types.ChartStatusPushed}, {Status: types.ChartStatusPushing}, {Status: types.ChartStatusError}}, false, true},
+		{"three charts, status pushed, pushing, and unknown", []types.Chart{{Status: types.ChartStatusPushed}, {Status: types.ChartStatusPushing}, {Status: types.ChartStatusUnknown}}, false, false},
+		{"three charts, status pushed, error, and unknown", []types.Chart{{Status: types.ChartStatusPushed}, {Status: types.ChartStatusError}, {Status: types.ChartStatusUnknown}}, false, true},
+		{"four charts, status pushed, pushing, error, and unknown", []types.Chart{{Status: types.ChartStatusPushed}, {Status: types.ChartStatusPushing}, {Status: types.ChartStatusError}, {Status: types.ChartStatusUnknown}}, false, true},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := areReleaseChartsPushed(tt.charts)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("areReleaseChartsReady() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			if got != tt.want {
+				t.Errorf("areReleaseChartsReady() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

cli/print/release.go

@@ -4,7 +4,7 @@ import (
 	"text/tabwriter"
 	"text/template"
 
-	releases "github.com/replicatedhq/replicated/gen/go/v1"
+	"github.com/replicatedhq/replicated/pkg/types"
 )
 
 var releaseTmplSrc = `SEQUENCE:	{{ .Sequence }}
@@ -16,7 +16,7 @@ CONFIG:
 
 var releaseTmpl = template.Must(template.New("Release").Funcs(funcs).Parse(releaseTmplSrc))
 
-func Release(w *tabwriter.Writer, release *releases.AppRelease) error {
+func Release(w *tabwriter.Writer, release *types.AppRelease) error {
 	if err := releaseTmpl.Execute(w, release); err != nil {
 		return err
 	}

client/release.go

@@ -3,7 +3,6 @@ package client
 import (
 	"errors"
 
-	releases "github.com/replicatedhq/replicated/gen/go/v1"
 	"github.com/replicatedhq/replicated/pkg/types"
 )
 
@@ -100,10 +99,20 @@ func (c *Client) TestRelease(appID string, appType string, sequence int64) (stri
 	return "", errors.New("unsupported app type")
 }
 
-func (c *Client) GetRelease(appID string, appType string, sequence int64) (*releases.AppRelease, error) {
+func (c *Client) GetRelease(appID string, appType string, sequence int64) (*types.AppRelease, error) {
 
 	if appType == "platform" {
-		return c.PlatformClient.GetRelease(appID, sequence)
+		release, err := c.PlatformClient.GetRelease(appID, sequence)
+		if err != nil {
+			return nil, err
+		}
+		return &types.AppRelease{
+			Config:    release.Config,
+			CreatedAt: release.CreatedAt,
+			Editable:  release.Editable,
+			EditedAt:  release.EditedAt,
+			Sequence:  release.Sequence,
+		}, nil
 	} else if appType == "kots" {
 		return c.KotsClient.GetRelease(appID, sequence)
 	}

pkg/kots/release/save.go

@@ -8,13 +8,13 @@ import (
 	"path/filepath"
 
 	"github.com/pkg/errors"
-	releases "github.com/replicatedhq/replicated/gen/go/v1"
-	"github.com/replicatedhq/replicated/pkg/kots/release/types"
+	releaseTypes "github.com/replicatedhq/replicated/pkg/kots/release/types"
 	"github.com/replicatedhq/replicated/pkg/logger"
+	"github.com/replicatedhq/replicated/pkg/types"
 )
 
-func Save(dstDir string, release *releases.AppRelease, log *logger.Logger) error {
-	var releaseYamls []types.KotsSingleSpec
+func Save(dstDir string, release *types.AppRelease, log *logger.Logger) error {
+	var releaseYamls []releaseTypes.KotsSingleSpec
 	if err := json.Unmarshal([]byte(release.Config), &releaseYamls); err != nil {
 		return errors.Wrap(err, "unmarshal release yamls")
 	}
@@ -31,7 +31,7 @@ func Save(dstDir string, release *releases.AppRelease, log *logger.Logger) error
 
 }
 
-func writeReleaseFiles(dstDir string, specs []types.KotsSingleSpec, log *logger.Logger) error {
+func writeReleaseFiles(dstDir string, specs []releaseTypes.KotsSingleSpec, log *logger.Logger) error {
 	for _, spec := range specs {
 		if len(spec.Children) > 0 {
 			err := writeReleaseDirectory(dstDir, spec, log)
@@ -49,7 +49,7 @@ func writeReleaseFiles(dstDir string, specs []types.KotsSingleSpec, log *logger.
 	return nil
 }
 
-func writeReleaseDirectory(dstDir string, spec types.KotsSingleSpec, log *logger.Logger) error {
+func writeReleaseDirectory(dstDir string, spec releaseTypes.KotsSingleSpec, log *logger.Logger) error {
 	log.ChildActionWithoutSpinner(spec.Path)
 
 	if err := os.Mkdir(filepath.Join(dstDir, spec.Path), 0755); err != nil && !os.IsExist(err) {
@@ -64,7 +64,7 @@ func writeReleaseDirectory(dstDir string, spec types.KotsSingleSpec, log *logger
 	return nil
 }
 
-func writeReleaseFile(dstDir string, spec types.KotsSingleSpec, log *logger.Logger) error {
+func writeReleaseFile(dstDir string, spec releaseTypes.KotsSingleSpec, log *logger.Logger) error {
 	log.ChildActionWithoutSpinner(spec.Path)
 
 	var content []byte

pkg/kots/release/save_test.go

@@ -8,8 +8,8 @@ import (
 	"testing"
 
 	"github.com/pkg/errors"
-	releases "github.com/replicatedhq/replicated/gen/go/v1"
 	"github.com/replicatedhq/replicated/pkg/logger"
+	"github.com/replicatedhq/replicated/pkg/types"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -26,7 +26,7 @@ func Test_Save(t *testing.T) {
 
 		defer os.RemoveAll(dstDir)
 
-		release := &releases.AppRelease{
+		release := &types.AppRelease{
 			Config: string(releaseToSave),
 		}
 		err = Save(dstDir, release, logger.NewLogger(os.Stdout))

pkg/kotsclient/release.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 
 	"github.com/pkg/errors"
-	releases "github.com/replicatedhq/replicated/gen/go/v1"
 	"github.com/replicatedhq/replicated/pkg/graphql"
 	"github.com/replicatedhq/replicated/pkg/types"
 )
@@ -44,7 +43,7 @@ func (c *VendorV3Client) TestRelease(appID string, sequence int64) (string, erro
 	return "", nil
 }
 
-func (c *VendorV3Client) GetRelease(appID string, sequence int64) (*releases.AppRelease, error) {
+func (c *VendorV3Client) GetRelease(appID string, sequence int64) (*types.AppRelease, error) {
 	resp := types.KotsGetReleaseResponse{}
 
 	path := fmt.Sprintf("/v3/app/%s/release/%v", appID, sequence)
@@ -54,11 +53,12 @@ func (c *VendorV3Client) GetRelease(appID string, sequence int64) (*releases.App
 		return nil, errors.Wrap(err, "failed to get release")
 	}
 
-	appRelease := releases.AppRelease{
+	appRelease := types.AppRelease{
 		Config:    resp.Release.Spec,
 		CreatedAt: resp.Release.CreatedAt,
 		Editable:  !resp.Release.IsReleaseNotEditable,
 		Sequence:  resp.Release.Sequence,
+		Charts:    resp.Release.Charts,
 	}
 
 	return &appRelease, nil

pkg/types/release.go

@@ -94,3 +94,12 @@ type EntitlementValue struct {
 	Name  string `json:"name,omitempty"`
 	Value string `json:"value,omitempty"`
 }
+
+type AppRelease struct {
+	Config    string    `json:"config,omitempty"`
+	CreatedAt time.Time `json:"createdAt,omitempty"`
+	Editable  bool      `json:"editable,omitempty"`
+	EditedAt  time.Time `json:"editedAt,omitempty"`
+	Sequence  int64     `json:"sequence,omitempty"`
+	Charts    []Chart   `json:"charts,omitempty"`
+}