APP-4292 viam-server increase stop timeout (#20)

cmd/viam-agent/main.go

@@ -159,7 +159,8 @@ func main() {
 			if errors.Is(err, agent.ErrSubsystemDisabled) {
 				globalLogger.Warn("provisioning subsystem disabled, please manually update /etc/viam.json and connect to internet")
 			} else {
-				globalLogger.Error(errors.Wrapf(err, "could not start provisioning subsystem, please manually update /etc/viam.json and connect to internet"))
+				globalLogger.Error(errors.Wrapf(err, "could not start provisioning subsystem"))
+				globalLogger.Error("please manually update /etc/viam.json and connect to internet")
 			}
 		}
 

manager.go

@@ -25,8 +25,11 @@ import (
 const (
 	minimalCheckInterval  = time.Second * 60
 	defaultNetworkTimeout = time.Second * 15
-	agentCachePath        = "agent-config.json"
-	SubsystemName         = "viam-agent"
+	// stopAllTimeout must be lower than systemd subsystems/viamagent/viam-agent.service timeout of 4mins
+	// and higher than subsystems/viamserver/viamserver.go timeout of 2mins.
+	stopAllTimeout = time.Minute * 3
+	agentCachePath = "agent-config.json"
+	SubsystemName  = "viam-agent"
 )
 
 // Manager is the core of the agent process, and maintains the list of subsystems, as well as cloud connection.
@@ -209,7 +212,7 @@ func (m *Manager) SubsystemHealthChecks(ctx context.Context) {
 
 // CloseAll stops all subsystems and closes the cloud connection.
 func (m *Manager) CloseAll() {
-	ctx, cancelFunc := context.WithTimeout(context.Background(), time.Minute)
+	ctx, cancelFunc := context.WithTimeout(context.Background(), stopAllTimeout)
 	defer cancelFunc()
 
 	m.subsystemsMu.Lock()

preinstall.sh

@@ -21,7 +21,7 @@ Type=exec
 Restart=always
 RestartSec=5
 User=root
-TimeoutSec=120
+TimeoutSec=240
 ExecStart=/opt/viam/bin/viam-agent --config /etc/viam.json
 FinalKillSignal=SIGQUIT
 

subsystem.go

@@ -23,9 +23,10 @@ import (
 )
 
 const (
-	ShortFailTime = time.Second * 30
-	StartTimeout  = time.Minute
-	StopTimeout   = time.Minute
+	ShortFailTime   = time.Second * 30
+	StartTimeout    = time.Minute
+	StopTermTimeout = time.Second * 30
+	StopKillTimeout = time.Second * 10
 )
 
 var ErrSubsystemDisabled = errors.New("subsystem disabled")
@@ -513,7 +514,7 @@ func (is *InternalSubsystem) Stop(ctx context.Context) error {
 		is.logger.Error(err)
 	}
 
-	if is.waitForExit(ctx, StopTimeout/2) {
+	if is.waitForExit(ctx, StopTermTimeout) {
 		is.logger.Infof("%s successfully stopped", is.name)
 		return nil
 	}
@@ -524,7 +525,7 @@ func (is *InternalSubsystem) Stop(ctx context.Context) error {
 		is.logger.Error(err)
 	}
 
-	if is.waitForExit(ctx, StopTimeout/2) {
+	if is.waitForExit(ctx, StopKillTimeout) {
 		is.logger.Infof("%s successfully killed", is.name)
 		return nil
 	}

subsystems/viamagent/viam-agent.service

@@ -8,7 +8,7 @@ Type=exec
 Restart=always
 RestartSec=5
 User=root
-TimeoutSec=120
+TimeoutSec=240
 RuntimeDirectory=viam
 RuntimeDirectoryMode=0755
 ExecStart=/opt/viam/bin/viam-agent --config /etc/viam.json

subsystems/viamserver/viamserver.go

@@ -28,8 +28,10 @@ func init() {
 
 const (
 	startTimeout = time.Minute * 5
-	stopTimeout  = time.Minute * 2
-	SubsysName   = "viam-server"
+	// stopTermTimeout must be higher than viam-server shutdown timeout of 90 secs.
+	stopTermTimeout = time.Minute * 2
+	stopKillTimeout = time.Second * 10
+	SubsysName      = "viam-server"
 )
 
 var (
@@ -160,7 +162,7 @@ func (s *viamServer) Stop(ctx context.Context) error {
 		s.logger.Error(err)
 	}
 
-	if s.waitForExit(ctx, stopTimeout/2) {
+	if s.waitForExit(ctx, stopTermTimeout) {
 		s.logger.Infof("%s successfully stopped", SubsysName)
 		return nil
 	}
@@ -171,7 +173,7 @@ func (s *viamServer) Stop(ctx context.Context) error {
 		s.logger.Error(err)
 	}
 
-	if s.waitForExit(ctx, stopTimeout/2) {
+	if s.waitForExit(ctx, stopKillTimeout) {
 		s.logger.Infof("%s successfully killed", SubsysName)
 		return nil
 	}