Fix the pending-upgrade problem (#1178)

* Playing with channels

* go mod tidy

* Using unknown status

* Adding check for timeout

* Fixing errors

* Add debug

* Adding debug

* Account for empty map

* Changing logs

* Adding recovery from pending to install

* Fixing return

* Testing simpler heuristic approach

* Removing stuff

* More cleanup

* Fixing timeout

* Adding some magic

* Updating status

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/prometheus/client_golang v1.20.5
 	github.com/spf13/afero v1.11.0
 	github.com/spf13/viper v1.19.0
+	helm.sh/helm/v3 v3.10.3
 	k8s.io/api v0.25.2
 	k8s.io/apimachinery v0.25.2
 	k8s.io/client-go v0.25.2
@@ -160,7 +161,6 @@ require (
 	gopkg.in/resty.v1 v1.12.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
-	helm.sh/helm/v3 v3.10.3 // indirect
 	k8s.io/apiextensions-apiserver v0.25.2 // indirect
 	k8s.io/apiserver v0.25.2 // indirect
 	k8s.io/cli-runtime v0.25.2 // indirect

service/controller/chart/resource/release/create.go

@@ -148,16 +148,13 @@ func (r *Resource) ApplyCreateChange(ctx context.Context, obj, createChange inte
 		}
 
 		r.logger.Debugf(ctx, "doing internal upgrade for release %#q", releaseState.Name)
-		e = hc.UpdateReleaseFromTarball(ctx,
+
+		err = hc.UpdateReleaseFromTarball(ctx,
 			tarballPath,
 			ns,
 			releaseState.Name,
 			releaseState.Values,
 			uOpts)
-
-		if e != nil {
-			err = e
-		}
 	}()
 
 	select {

service/controller/chart/resource/release/update.go

@@ -6,11 +6,16 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/giantswarm/apiextensions-application/api/v1alpha1"
 	"github.com/giantswarm/helmclient/v4/pkg/helmclient"
 	"github.com/giantswarm/microerror"
 	"github.com/giantswarm/operatorkit/v7/pkg/controller/context/resourcecanceledcontext"
 	"github.com/giantswarm/operatorkit/v7/pkg/resource/crud"
 	"github.com/google/go-cmp/cmp"
+	"helm.sh/helm/v3/pkg/release"
+	"helm.sh/helm/v3/pkg/storage"
+	"helm.sh/helm/v3/pkg/storage/driver"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
 	"github.com/giantswarm/chart-operator/v3/pkg/annotation"
 	"github.com/giantswarm/chart-operator/v3/pkg/project"
@@ -117,8 +122,8 @@ func (r *Resource) ApplyUpdateChange(ctx context.Context, obj, updateChange inte
 			releaseState.Name,
 			releaseState.Values,
 			opts)
-		close(ch)
 
+		close(ch)
 		/*
 			The two-step installation is not supported on upgrades, yet it could be
 			useful in times when app is already installed in version A, and is being
@@ -283,6 +288,8 @@ func (r *Resource) newUpdateChange(ctx context.Context, obj, currentState, desir
 			// no-op after rollback
 			return nil, nil
 		}
+
+		r.tryRecoverFromPending(ctx, cr, &currentReleaseState)
 	}
 
 	// We check the controller context and if the release has failed more
@@ -386,3 +393,54 @@ func (r *Resource) rollback(ctx context.Context, obj interface{}, currentStatus
 
 	return nil
 }
+
+func (r *Resource) tryRecoverFromPending(ctx context.Context, cr v1alpha1.Chart, rs *ReleaseState) {
+	s := driver.NewSecrets(r.k8sClient.CoreV1().Secrets(key.Namespace(cr)))
+	store := storage.Init(s)
+
+	rel, err := store.Last(rs.Name)
+	if err != nil {
+		r.logger.Debugf(ctx, "encountered error on getting last revision for release %#q", rs.Name)
+		return
+	}
+
+	timeout := getTimeout(cr, rel.Version)
+
+	if time.Since(rel.Info.LastDeployed.Time) < (*timeout).Duration {
+		r.logger.Debugf(ctx, "timeout has not elapsed yet for release %#q, skipping recevery", rs.Name)
+		return
+	}
+
+	status := release.StatusUnknown
+	if rel.Version == 1 {
+		status = release.StatusFailed
+	}
+	rel.SetStatus(status, "Chart Operator has been restarted when performing Helm operation")
+	err = store.Update(rel)
+	if err != nil {
+		r.logger.Debugf(ctx, "encountered error on updating status for release %#q", rs.Name)
+		return
+	}
+
+	rs.Status = string(release.StatusUnknown)
+
+	r.logger.Debugf(ctx, "recovered from `pending` status for release %#q", rs.Name)
+}
+
+func getTimeout(cr v1alpha1.Chart, rev int) *metav1.Duration {
+	timeout := &metav1.Duration{Duration: 5 * time.Minute}
+
+	if rev == 1 {
+		installTimeout := key.InstallTimeout(cr)
+		if installTimeout != nil {
+			timeout = installTimeout
+		}
+	} else {
+		upgradeTimeout := key.UpgradeTimeout(cr)
+		if upgradeTimeout != nil {
+			timeout = upgradeTimeout
+		}
+	}
+
+	return timeout
+}