feat: added retry on files sync error (GoogleContainerTools#9261)

* feat: added retry on files sync error * reverted code style changes * fix(dev.go): change package order Signed-off-by: Suleiman Dibirov <[email protected]> --------- Signed-off-by: Suleiman Dibirov <[email protected]>
idsulik · Jul 12, 2024 · 2cbeb78 · 2cbeb78
1 parent 2393d9d
commit 2cbeb78
Showing 1 changed file with 21 additions and 2 deletions.
diff --git a/pkg/skaffold/runner/dev.go b/pkg/skaffold/runner/dev.go
@@ -23,6 +23,8 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/cenkalti/backoff/v4"
+
 	"github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/constants"
 	"github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/event"
 	eventV2 "github.com/GoogleContainerTools/skaffold/v2/pkg/skaffold/event/v2"
@@ -85,9 +87,13 @@ func (r *SkaffoldRunner) doDev(ctx context.Context, out io.Writer) error {
 			r.changeSet.ResetSync()
 			r.intents.ResetSync()
 		}()
+
+		// todo: make this configurable
+		opts := backoff.WithMaxRetries(backoff.NewExponentialBackOff(), 3)
 		instrumentation.AddDevIteration("sync")
 		meterUpdated = true
-		for _, s := range r.changeSet.NeedsResync() {
+
+		syncHandler := func(s *sync.Item) error {
 			fileCount := len(s.Copy) + len(s.Delete)
 			output.Default.Fprintf(out, "Syncing %d files for %s\n", fileCount, s.Image)
 			fileSyncInProgress(fileCount, s.Image)
@@ -99,10 +105,23 @@ func (r *SkaffoldRunner) doDev(ctx context.Context, out io.Writer) error {
 				eventV2.TaskFailed(constants.DevLoop, err)
 				endTrace(instrumentation.TraceEndError(err))
 
-				return nil
+				return err
 			}
 
 			fileSyncSucceeded(fileCount, s.Image)
+
+			return nil
+		}
+		for _, s := range r.changeSet.NeedsResync() {
+			err := backoff.Retry(
+				func() error {
+					return syncHandler(s)
+				}, backoff.WithContext(opts, childCtx),
+			)
+
+			if err != nil {
+				return nil
+			}
 		}
 		endTrace()
 	}