Adding parameter to allow containers to be updated only after some number of days have passed since the new image has been published

cmd/root.go

@@ -39,6 +39,7 @@ var (
 	disableContainers []string
 	notifier          t.Notifier
 	timeout           time.Duration
+	deferDays         int
 	lifecycleHooks    bool
 	rollingRestart    bool
 	scope             string
@@ -96,6 +97,7 @@ func PreRun(cmd *cobra.Command, _ []string) {
 
 	enableLabel, _ = f.GetBool("label-enable")
 	disableContainers, _ = f.GetStringSlice("disable-containers")
+	deferDays, _ = f.GetInt("defer-days")
 	lifecycleHooks, _ = f.GetBool("enable-lifecycle-hooks")
 	rollingRestart, _ = f.GetBool("rolling-restart")
 	scope, _ = f.GetString("scope")
@@ -288,6 +290,10 @@ func writeStartupMessage(c *cobra.Command, sched time.Time, filtering string) {
 		until := formatDuration(time.Until(sched))
 		startupLog.Info("Scheduling first run: " + sched.Format("2006-01-02 15:04:05 -0700 MST"))
 		startupLog.Info("Note that the first check will be performed in " + until)
+		deferDays, _ = c.PersistentFlags().GetInt("defer-days")
+		if deferDays > 0 {
+			startupLog.Infof("Container updates will be deferred until %d day(s) after image creation.", deferDays)
+		}
 	} else if runOnce, _ := c.PersistentFlags().GetBool("run-once"); runOnce {
 		startupLog.Info("Running a one time update.")
 	} else {
@@ -364,6 +370,7 @@ func runUpdatesWithNotifications(filter t.Filter) *metrics.Metric {
 		NoRestart:       noRestart,
 		Timeout:         timeout,
 		MonitorOnly:     monitorOnly,
+		DeferDays:       deferDays,
 		LifecycleHooks:  lifecycleHooks,
 		RollingRestart:  rollingRestart,
 		LabelPrecedence: labelPrecedence,
@@ -376,9 +383,10 @@ func runUpdatesWithNotifications(filter t.Filter) *metrics.Metric {
 	notifier.SendNotification(result)
 	metricResults := metrics.NewMetric(result)
 	notifications.LocalLog.WithFields(log.Fields{
-		"Scanned": metricResults.Scanned,
-		"Updated": metricResults.Updated,
-		"Failed":  metricResults.Failed,
+		"Scanned":  metricResults.Scanned,
+		"Updated":  metricResults.Updated,
+		"Deferred": metricResults.Deferred,
+		"Failed":   metricResults.Failed,
 	}).Info("Session done")
 	return metricResults
 }

docs/arguments.md

@@ -381,6 +381,16 @@ Environment Variable: WATCHTOWER_HTTP_API_METRICS
              Default: false
 ```
 
+## Deferred Update
+Only update container to latest version of image if some number of days have passed since it has been published. This option may be useful for those who wish to avoid updating prior to the new version having some time in the field prior to updating in case there are critical defects found and released in a subsequent version.
+
+```text
+            Argument: --defer-days
+Environment Variable: WATCHTOWER_DEFER_DAYS
+                Type: Integer
+             Default: false
+```
+
 ## Scheduling
 [Cron expression](https://pkg.go.dev/github.com/robfig/[email protected]?tab=doc#hdr-CRON_Expression_Format) in 6 fields (rather than the traditional 5) which defines when and how often to check for new images. Either `--interval` or the schedule expression
 can be defined, but not both. An example: `--schedule "0 0 4 * * *"`

internal/actions/mocks/container.go

@@ -79,6 +79,13 @@ func CreateMockContainerWithDigest(id string, name string, image string, created
 	return c
 }
 
+// CreateMockContainerWithDigest should only be used for testing
+func CreateMockContainerWithImageCreatedTime(id string, name string, image string, created time.Time, imageCreated time.Time) wt.Container {
+	c := CreateMockContainer(id, name, image, created)
+	c.ImageInfo().Created = imageCreated.UTC().Format(time.RFC3339Nano)
+	return c
+}
+
 // CreateMockContainerWithConfig creates a container substitute valid for testing
 func CreateMockContainerWithConfig(id string, name string, image string, running bool, restarting bool, created time.Time, config *dockerContainer.Config) wt.Container {
 	content := types.ContainerJSON{

internal/actions/update.go

@@ -2,6 +2,7 @@ package actions
 
 import (
 	"errors"
+	"time"
 
 	"github.com/containrrr/watchtower/internal/util"
 	"github.com/containrrr/watchtower/pkg/container"
@@ -33,13 +34,24 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 	staleCheckFailed := 0
 
 	for i, targetContainer := range containers {
+		// stale will be true if there is a more recent image than the current container is using
 		stale, newestImage, err := client.IsContainerStale(targetContainer, params)
 		shouldUpdate := stale && !params.NoRestart && !targetContainer.IsMonitorOnly(params)
+		imageUpdateDeferred := false
+		imageAgeDays := 0
 		if err == nil && shouldUpdate {
-			// Check to make sure we have all the necessary information for recreating the container
+			// Check to make sure we have all the necessary information for recreating the container, including ImageInfo
 			err = targetContainer.VerifyConfiguration()
-			// If the image information is incomplete and trace logging is enabled, log it for further diagnosis
-			if err != nil && log.IsLevelEnabled(log.TraceLevel) {
+			if err == nil {
+				if params.DeferDays > 0 {
+					imageAgeDays, imageErr := getImageAgeDays(targetContainer.ImageInfo().Created)
+					err = imageErr
+					if err == nil {
+						imageUpdateDeferred = imageAgeDays < params.DeferDays
+					}
+				}
+			} else if log.IsLevelEnabled(log.TraceLevel) {
+				// If the image information is incomplete and trace logging is enabled, log it for further diagnosis
 				imageInfo := targetContainer.ImageInfo()
 				log.Tracef("Image info: %#v", imageInfo)
 				log.Tracef("Container info: %#v", targetContainer.ContainerInfo())
@@ -54,6 +66,12 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 			stale = false
 			staleCheckFailed++
 			progress.AddSkipped(targetContainer, err)
+		} else if imageUpdateDeferred {
+			log.Infof("New image found for %s that was created %d day(s) ago but update deferred until %d day(s) after creation", targetContainer.Name(), imageAgeDays, params.DeferDays)
+			// technically the container is stale but we set it to false here because it is this stale flag that tells downstream methods whether to perform the update
+			stale = false
+			progress.AddScanned(targetContainer, newestImage)
+			progress.MarkDeferred(targetContainer.ID())
 		} else {
 			progress.AddScanned(targetContainer, newestImage)
 		}
@@ -71,9 +89,13 @@ func Update(client container.Client, params types.UpdateParams) (types.Report, e
 
 	UpdateImplicitRestart(containers)
 
+	// containersToUpdate will contain all containers, not just those that need to be updated. The "stale" flag is checked via container.ToRestart()
+	// within stopContainersInReversedOrder and restartContainersInSortedOrder to skip containers with stale set to false (unless LinkedToRestarting set)
+	// NOTE: This logic is changing with latest PR on main repo
 	var containersToUpdate []types.Container
 	for _, c := range containers {
-		if !c.IsMonitorOnly(params) {
+		// pulling this change in from PR 1895 for now to avoid updating status incorrectly
+		if c.ToRestart() && !c.IsMonitorOnly(params) {
 			containersToUpdate = append(containersToUpdate, c)
 			progress.MarkForUpdate(c.ID())
 		}
@@ -265,3 +287,18 @@ func linkedContainerMarkedForRestart(links []string, containers []types.Containe
 	}
 	return ""
 }
+
+// Finds the difference between now and a given date, in full days. Input date is expected to originate
+// from an image's Created attribute which will follow ISO 3339/8601 format.
+// Reference: https://docs.docker.com/engine/api/v1.43/#tag/Image/operation/ImageInspect
+func getImageAgeDays(imageCreatedDateTime string) (int, error) {
+	imageCreatedDate, error := time.Parse(time.RFC3339Nano, imageCreatedDateTime)
+
+	if error != nil {
+		log.Errorf("Error parsing imageCreatedDateTime date (%s). Error: %s", imageCreatedDateTime, error)
+		return -1, error
+	}
+
+	return int(time.Since(imageCreatedDate).Hours() / 24), nil
+
+}

internal/actions/update_test.go

@@ -65,6 +65,35 @@ func getLinkedTestData(withImageInfo bool) *TestData {
 	}
 }
 
+func getMixedAgeTestData(keepContainer string) *TestData {
+	return &TestData{
+		NameOfContainerToKeep: keepContainer,
+		Containers: []types.Container{
+			// new container with 5 day old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-01",
+				"test-container-01",
+				"fake-image-01:latest",
+				time.Now(),
+				time.Now().AddDate(0, 0, -5)),
+			// new container with 1 day old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-02",
+				"test-container-02",
+				"fake-image-02:latest",
+				time.Now(),
+				time.Now().AddDate(0, 0, -1)),
+			// new container with 1 hour old image
+			CreateMockContainerWithImageCreatedTime(
+				"test-container-03",
+				"test-container-03",
+				"fake-image-03:latest",
+				time.Now(),
+				time.Now().Add(-1*time.Hour)),
+		},
+	}
+}
+
 var _ = Describe("the update action", func() {
 	When("watchtower has been instructed to clean up", func() {
 		When("there are multiple containers using the same image", func() {
@@ -258,6 +287,37 @@ var _ = Describe("the update action", func() {
 		})
 	})
 
+	When("watchtower has been instructed to defer updates by some number of days", func() {
+		It("should only update the 1 container with image at least 2 days old when DeferDays is 2", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 2})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(1))
+			Expect(report.Deferred()).To(HaveLen(2))
+		})
+		It("should only update the 2 containers with image at least 1 day old when DeferDays is 1", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 1})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(2))
+			Expect(report.Deferred()).To(HaveLen(1))
+		})
+		It("should update all containers when DeferDays is 0", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{DeferDays: 0})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(3))
+			Expect(report.Deferred()).To(HaveLen(0))
+		})
+		It("should update all containers when DeferDays is not specified", func() {
+			client := CreateMockClient(getMixedAgeTestData(""), false, false)
+			report, err := actions.Update(client, types.UpdateParams{})
+			Expect(err).NotTo(HaveOccurred())
+			Expect(report.Updated()).To(HaveLen(3))
+			Expect(report.Deferred()).To(HaveLen(0))
+		})
+	})
+
 	When("watchtower has been instructed to run lifecycle hooks", func() {
 
 		When("pre-update script returns 1", func() {

internal/flags/flags.go

@@ -147,6 +147,12 @@ func RegisterSystemFlags(rootCmd *cobra.Command) {
 		envBool("WATCHTOWER_LIFECYCLE_HOOKS"),
 		"Enable the execution of commands triggered by pre- and post-update lifecycle hooks")
 
+	flags.IntP(
+		"defer-days",
+		"0",
+		envInt("WATCHTOWER_DEFER_DAYS"),
+		"Number of days to wait for new image version to be in place prior to installing it")
+
 	flags.BoolP(
 		"rolling-restart",
 		"",

internal/flags/flags_test.go

@@ -278,9 +278,9 @@ func TestProcessFlagAliasesInvalidPorcelaineVersion(t *testing.T) {
 	})
 }
 
-func TestFlagsArePrecentInDocumentation(t *testing.T) {
+func TestFlagsArePresentInDocumentation(t *testing.T) {
 
-	// Legacy notifcations are ignored, since they are (soft) deprecated
+	// Legacy notifications are ignored, since they are (soft) deprecated
 	ignoredEnvs := map[string]string{
 		"WATCHTOWER_NOTIFICATION_SLACK_ICON_EMOJI": "legacy",
 		"WATCHTOWER_NOTIFICATION_SLACK_ICON_URL":   "legacy",

pkg/metrics/metrics.go

@@ -10,28 +10,31 @@ var metrics *Metrics
 
 // Metric is the data points of a single scan
 type Metric struct {
-	Scanned int
-	Updated int
-	Failed  int
+	Scanned  int
+	Updated  int
+	Deferred int
+	Failed   int
 }
 
 // Metrics is the handler processing all individual scan metrics
 type Metrics struct {
-	channel chan *Metric
-	scanned prometheus.Gauge
-	updated prometheus.Gauge
-	failed  prometheus.Gauge
-	total   prometheus.Counter
-	skipped prometheus.Counter
+	channel  chan *Metric
+	scanned  prometheus.Gauge
+	updated  prometheus.Gauge
+	deferred prometheus.Gauge
+	failed   prometheus.Gauge
+	total    prometheus.Counter
+	skipped  prometheus.Counter
 }
 
 // NewMetric returns a Metric with the counts taken from the appropriate types.Report fields
 func NewMetric(report types.Report) *Metric {
 	return &Metric{
 		Scanned: len(report.Scanned()),
 		// Note: This is for backwards compatibility. ideally, stale containers should be counted separately
-		Updated: len(report.Updated()) + len(report.Stale()),
-		Failed:  len(report.Failed()),
+		Updated:  len(report.Updated()) + len(report.Stale()),
+		Deferred: len(report.Deferred()),
+		Failed:   len(report.Failed()),
 	}
 }
 
@@ -60,6 +63,10 @@ func Default() *Metrics {
 			Name: "watchtower_containers_updated",
 			Help: "Number of containers updated by watchtower during the last scan",
 		}),
+		deferred: promauto.NewGauge(prometheus.GaugeOpts{
+			Name: "watchtower_containers_deferred",
+			Help: "Number of containers deferred by watchtower during the last scan",
+		}),
 		failed: promauto.NewGauge(prometheus.GaugeOpts{
 			Name: "watchtower_containers_failed",
 			Help: "Number of containers where update failed during the last scan",
@@ -95,13 +102,15 @@ func (metrics *Metrics) HandleUpdate(channel <-chan *Metric) {
 			metrics.skipped.Inc()
 			metrics.scanned.Set(0)
 			metrics.updated.Set(0)
+			metrics.deferred.Set(0)
 			metrics.failed.Set(0)
 			continue
 		}
 		// Update metrics with the new values
 		metrics.total.Inc()
 		metrics.scanned.Set(float64(change.Scanned))
 		metrics.updated.Set(float64(change.Updated))
+		metrics.deferred.Set(float64(change.Deferred))
 		metrics.failed.Set(float64(change.Failed))
 	}
 }

pkg/notifications/json.go

@@ -23,12 +23,13 @@ func (d Data) MarshalJSON() ([]byte, error) {
 	var report jsonMap
 	if d.Report != nil {
 		report = jsonMap{
-			`scanned`: marshalReports(d.Report.Scanned()),
-			`updated`: marshalReports(d.Report.Updated()),
-			`failed`:  marshalReports(d.Report.Failed()),
-			`skipped`: marshalReports(d.Report.Skipped()),
-			`stale`:   marshalReports(d.Report.Stale()),
-			`fresh`:   marshalReports(d.Report.Fresh()),
+			`scanned`:  marshalReports(d.Report.Scanned()),
+			`updated`:  marshalReports(d.Report.Updated()),
+			`deferred`: marshalReports(d.Report.Deferred()),
+			`failed`:   marshalReports(d.Report.Failed()),
+			`skipped`:  marshalReports(d.Report.Skipped()),
+			`stale`:    marshalReports(d.Report.Stale()),
+			`fresh`:    marshalReports(d.Report.Fresh()),
 		}
 	}
 

pkg/notifications/json_test.go

@@ -21,6 +21,7 @@ var _ = Describe("JSON template", func() {
 		],
 		"host": "Mock",
 		"report": {
+		"deferred": [],
 		"failed": [
 			{
 				"currentImageId": "01d210000000",
@@ -110,7 +111,7 @@ var _ = Describe("JSON template", func() {
 		},
 		"title": "Watchtower updates on Mock"
 }`
-				data := mockDataFromStates(s.UpdatedState, s.FreshState, s.FailedState, s.SkippedState, s.UpdatedState)
+				data := mockDataFromStates(s.UpdatedState, s.DeferredState, s.FreshState, s.FailedState, s.SkippedState, s.UpdatedState)
 				Expect(getTemplatedResult(`json.v1`, false, data)).To(MatchJSON(expected))
 			})
 		})

pkg/notifications/preview/data/data.go

@@ -72,6 +72,8 @@ func (pb *previewData) addContainer(c containerStatus) {
 		pb.report.scanned = append(pb.report.scanned, &c)
 	case UpdatedState:
 		pb.report.updated = append(pb.report.updated, &c)
+	case DeferredState:
+		pb.report.deferred = append(pb.report.deferred, &c)
 	case FailedState:
 		pb.report.failed = append(pb.report.failed, &c)
 	case SkippedState: