From ec8801225f69b09d30ceb0cc2d66b08b15963a4e Mon Sep 17 00:00:00 2001 From: Masayuki Ishii Date: Thu, 25 Jul 2024 14:27:28 +0900 Subject: [PATCH 1/2] Set timeout to FW update commands Signed-off-by: Masayuki Ishii --- pkg/neco/cmd/apply-firmware.go | 10 ++++++++-- pkg/neco/cmd/isoreboot.go | 5 ++++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/pkg/neco/cmd/apply-firmware.go b/pkg/neco/cmd/apply-firmware.go index b2011ee60..d2026034d 100644 --- a/pkg/neco/cmd/apply-firmware.go +++ b/pkg/neco/cmd/apply-firmware.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sort" "sync" + "time" "github.com/cybozu-go/log" "github.com/cybozu-go/neco" @@ -34,14 +35,15 @@ If some nodes are already powered off, this command does not do anything to thos var applyFirmwareGetOpts sabakanMachinesGetOpts var applyFirmwareRebootOption bool +var applyFirmwareTimeoutOption time.Duration func applyFirmwareRun(cmd *cobra.Command, args []string) { ctx := context.Background() - uploadAssetsAndRunCommandOnWorkers(ctx, &applyFirmwareGetOpts, args, []string{"docker", "exec", "setup-hw", "setup-apply-firmware"}, applyFirmwareRebootOption) + uploadAssetsAndRunCommandOnWorkers(ctx, &applyFirmwareGetOpts, args, []string{"docker", "exec", "setup-hw", "setup-apply-firmware"}, applyFirmwareTimeoutOption, applyFirmwareRebootOption) } -func uploadAssetsAndRunCommandOnWorkers(ctx context.Context, getOpts *sabakanMachinesGetOpts, filenames []string, cmdline []string, needReboot bool) { +func uploadAssetsAndRunCommandOnWorkers(ctx context.Context, getOpts *sabakanMachinesGetOpts, filenames []string, cmdline []string, timeout time.Duration, needReboot bool) { machines, err := sabakanMachinesGet(ctx, getOpts) if err != nil { log.ErrorExit(err) @@ -94,6 +96,9 @@ func uploadAssetsAndRunCommandOnWorkers(ctx context.Context, getOpts *sabakanMac cmdArgs := []string{"ssh", addr} cmdArgs = append(cmdArgs, cmdline...) cmdArgs = append(cmdArgs, assetUrls...) + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() output, err := well.CommandContext(ctx, neco.CKECLIBin, cmdArgs...).Output() mtx.Lock() @@ -159,4 +164,5 @@ func init() { rootCmd.AddCommand(applyFirmwareCmd) addSabakanMachinesGetOpts(applyFirmwareCmd, &applyFirmwareGetOpts) applyFirmwareCmd.Flags().BoolVar(&applyFirmwareRebootOption, "reboot", false, "Schedule reboot") + applyFirmwareCmd.Flags().DurationVar(&applyFirmwareTimeoutOption, "timeout", 5*time.Minute, "timeout") } diff --git a/pkg/neco/cmd/isoreboot.go b/pkg/neco/cmd/isoreboot.go index a710f8a57..be7e9ea29 100644 --- a/pkg/neco/cmd/isoreboot.go +++ b/pkg/neco/cmd/isoreboot.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "time" "github.com/spf13/cobra" ) @@ -20,14 +21,16 @@ If some nodes are already powered off, this command does not do anything to thos } var isoRebootGetOpts sabakanMachinesGetOpts +var isoRebootTimeoutOption time.Duration func isoRebootRun(cmd *cobra.Command, args []string) { ctx := context.Background() - uploadAssetsAndRunCommandOnWorkers(ctx, &isoRebootGetOpts, args, []string{"docker", "exec", "setup-hw", "setup-isoreboot"}, true) + uploadAssetsAndRunCommandOnWorkers(ctx, &isoRebootGetOpts, args, []string{"docker", "exec", "setup-hw", "setup-isoreboot"}, isoRebootTimeoutOption, true) } func init() { rootCmd.AddCommand(isoRebootCmd) addSabakanMachinesGetOpts(isoRebootCmd, &isoRebootGetOpts) + isoRebootCmd.Flags().DurationVar(&isoRebootTimeoutOption, "timeout", 30*time.Second, "timeout") } From 010703b9a77fb5dc4313ca3d057b491238e69b67 Mon Sep 17 00:00:00 2001 From: YZ775 Date: Thu, 25 Jul 2024 06:15:03 +0000 Subject: [PATCH 2/2] extend ginkgo timeout to 90min --- dctest/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dctest/Makefile b/dctest/Makefile index d3c7c2c3d..cf6c24cc4 100644 --- a/dctest/Makefile +++ b/dctest/Makefile @@ -4,7 +4,7 @@ SUDO = sudo OUTPUT = ./output PLACEMAT = /usr/bin/placemat2 -GINKGO = env GO111MODULE=on $(GOPATH)/bin/ginkgo --fail-fast -v +GINKGO = env GO111MODULE=on $(GOPATH)/bin/ginkgo --fail-fast -v --timeout=90m SUITE = bootstrap PLACEMAT_DATADIR = /var/scratch/placemat VM_SHARED_DIR = /mnt/placemat