diff --git a/dctest/Makefile b/dctest/Makefile index d3c7c2c3d..cf6c24cc4 100644 --- a/dctest/Makefile +++ b/dctest/Makefile @@ -4,7 +4,7 @@ SUDO = sudo OUTPUT = ./output PLACEMAT = /usr/bin/placemat2 -GINKGO = env GO111MODULE=on $(GOPATH)/bin/ginkgo --fail-fast -v +GINKGO = env GO111MODULE=on $(GOPATH)/bin/ginkgo --fail-fast -v --timeout=90m SUITE = bootstrap PLACEMAT_DATADIR = /var/scratch/placemat VM_SHARED_DIR = /mnt/placemat diff --git a/pkg/neco/cmd/apply-firmware.go b/pkg/neco/cmd/apply-firmware.go index b2011ee60..d2026034d 100644 --- a/pkg/neco/cmd/apply-firmware.go +++ b/pkg/neco/cmd/apply-firmware.go @@ -8,6 +8,7 @@ import ( "path/filepath" "sort" "sync" + "time" "github.com/cybozu-go/log" "github.com/cybozu-go/neco" @@ -34,14 +35,15 @@ If some nodes are already powered off, this command does not do anything to thos var applyFirmwareGetOpts sabakanMachinesGetOpts var applyFirmwareRebootOption bool +var applyFirmwareTimeoutOption time.Duration func applyFirmwareRun(cmd *cobra.Command, args []string) { ctx := context.Background() - uploadAssetsAndRunCommandOnWorkers(ctx, &applyFirmwareGetOpts, args, []string{"docker", "exec", "setup-hw", "setup-apply-firmware"}, applyFirmwareRebootOption) + uploadAssetsAndRunCommandOnWorkers(ctx, &applyFirmwareGetOpts, args, []string{"docker", "exec", "setup-hw", "setup-apply-firmware"}, applyFirmwareTimeoutOption, applyFirmwareRebootOption) } -func uploadAssetsAndRunCommandOnWorkers(ctx context.Context, getOpts *sabakanMachinesGetOpts, filenames []string, cmdline []string, needReboot bool) { +func uploadAssetsAndRunCommandOnWorkers(ctx context.Context, getOpts *sabakanMachinesGetOpts, filenames []string, cmdline []string, timeout time.Duration, needReboot bool) { machines, err := sabakanMachinesGet(ctx, getOpts) if err != nil { log.ErrorExit(err) @@ -94,6 +96,9 @@ func uploadAssetsAndRunCommandOnWorkers(ctx context.Context, getOpts *sabakanMac cmdArgs := []string{"ssh", addr} cmdArgs = append(cmdArgs, cmdline...) cmdArgs = append(cmdArgs, assetUrls...) + + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() output, err := well.CommandContext(ctx, neco.CKECLIBin, cmdArgs...).Output() mtx.Lock() @@ -159,4 +164,5 @@ func init() { rootCmd.AddCommand(applyFirmwareCmd) addSabakanMachinesGetOpts(applyFirmwareCmd, &applyFirmwareGetOpts) applyFirmwareCmd.Flags().BoolVar(&applyFirmwareRebootOption, "reboot", false, "Schedule reboot") + applyFirmwareCmd.Flags().DurationVar(&applyFirmwareTimeoutOption, "timeout", 5*time.Minute, "timeout") } diff --git a/pkg/neco/cmd/isoreboot.go b/pkg/neco/cmd/isoreboot.go index a710f8a57..be7e9ea29 100644 --- a/pkg/neco/cmd/isoreboot.go +++ b/pkg/neco/cmd/isoreboot.go @@ -2,6 +2,7 @@ package cmd import ( "context" + "time" "github.com/spf13/cobra" ) @@ -20,14 +21,16 @@ If some nodes are already powered off, this command does not do anything to thos } var isoRebootGetOpts sabakanMachinesGetOpts +var isoRebootTimeoutOption time.Duration func isoRebootRun(cmd *cobra.Command, args []string) { ctx := context.Background() - uploadAssetsAndRunCommandOnWorkers(ctx, &isoRebootGetOpts, args, []string{"docker", "exec", "setup-hw", "setup-isoreboot"}, true) + uploadAssetsAndRunCommandOnWorkers(ctx, &isoRebootGetOpts, args, []string{"docker", "exec", "setup-hw", "setup-isoreboot"}, isoRebootTimeoutOption, true) } func init() { rootCmd.AddCommand(isoRebootCmd) addSabakanMachinesGetOpts(isoRebootCmd, &isoRebootGetOpts) + isoRebootCmd.Flags().DurationVar(&isoRebootTimeoutOption, "timeout", 30*time.Second, "timeout") }