fix(smoke): add timeout for upload and download

config/config.yaml

@@ -297,6 +297,8 @@ checks:
       postage-depth: 20
       nodes-sync-wait: 1m
       duration: 12h
+      upload-timeout: 5m
+      download-timeout: 5m
     timeout: 5m
     type: smoke
   load:

config/local.yaml

@@ -340,6 +340,8 @@ checks:
       postage-amount: 1000000
       postage-depth: 20
       nodes-sync-wait: 1m
+      upload-timeout: 1m
+      download-timeout: 1m
     timeout: 5m
     type: smoke
   ci-soc:

go.mod

@@ -53,12 +53,9 @@ require (
 	github.com/go-stack/stack v1.8.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
-	github.com/golang/snappy v0.0.4 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa // indirect
 	github.com/googleapis/gnostic v0.5.5 // indirect
-	github.com/hashicorp/errwrap v1.0.0 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.13 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
@@ -102,7 +99,6 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
-	github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
 	github.com/tklauser/go-sysconf v0.3.6 // indirect
 	github.com/tklauser/numcpus v0.2.2 // indirect
 	github.com/uber/jaeger-lib v2.2.0+incompatible // indirect

go.sum

@@ -208,7 +208,6 @@ github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw
 github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
 github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
@@ -257,11 +256,7 @@ github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
-github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
 github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE=
-github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
-github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs=
@@ -472,7 +467,6 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o
 github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8=
 github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0=
 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY=
-github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
 github.com/tklauser/go-sysconf v0.3.6 h1:oc1sJWvKkmvIxhDHeKWvZS4f6AW+YcoguSfRF2/Hmo4=
 github.com/tklauser/go-sysconf v0.3.6/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI=
 github.com/tklauser/numcpus v0.2.2 h1:oyhllyrScuYI6g+h/zUvNXNp1wy7x8qQy3t/piefldA=
@@ -589,7 +583,6 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
@@ -666,7 +659,6 @@ golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

pkg/check/smoke/load.go

@@ -59,7 +59,7 @@ func (c *LoadCheck) Run(ctx context.Context, cluster orchestration.Cluster, opts
 	ctx, cancel := context.WithTimeout(ctx, o.Duration)
 	defer cancel()
 
-	test := &test{opt: o, ctx: ctx, clients: clients, logger: c.logger}
+	test := &test{clients: clients, logger: c.logger}
 
 	uploaders := selectNames(cluster, o.UploadGroups...)
 	downloaders := selectNames(cluster, o.DownloadGroups...)
@@ -125,7 +125,7 @@ func (c *LoadCheck) Run(ctx context.Context, cluster orchestration.Cluster, opts
 						batches.Store(txName, batchID)
 					}
 
-					address, duration, err = test.uploadWithBatch(txName, txData, batchID)
+					address, duration, err = test.upload(ctx, txName, txData, batchID)
 					if err != nil {
 						c.metrics.UploadErrors.Inc()
 						c.logger.Infof("upload failed: %v", err)
@@ -174,7 +174,7 @@ func (c *LoadCheck) Run(ctx context.Context, cluster orchestration.Cluster, opts
 
 					c.metrics.DownloadAttempts.Inc()
 
-					rxData, rxDuration, err = test.download(rxName, address)
+					rxData, rxDuration, err = test.download(ctx, rxName, address)
 					if err != nil {
 						c.metrics.DownloadErrors.Inc()
 						c.logger.Infof("download failed: %v", err)

pkg/check/smoke/smoke.go

@@ -18,14 +18,16 @@ import (
 
 // Options represents smoke test options
 type Options struct {
-	ContentSize   int64
-	RndSeed       int64
-	PostageAmount int64
-	PostageDepth  uint64
-	TxOnErrWait   time.Duration
-	RxOnErrWait   time.Duration
-	NodesSyncWait time.Duration
-	Duration      time.Duration
+	ContentSize     int64
+	RndSeed         int64
+	PostageAmount   int64
+	PostageDepth    uint64
+	TxOnErrWait     time.Duration
+	RxOnErrWait     time.Duration
+	NodesSyncWait   time.Duration
+	Duration        time.Duration
+	UploadTimeout   time.Duration
+	DownloadTimeout time.Duration
 	// load test params
 	UploaderCount   int
 	UploadGroups    []string
@@ -38,16 +40,18 @@ type Options struct {
 // NewDefaultOptions returns new default options
 func NewDefaultOptions() Options {
 	return Options{
-		ContentSize:   5000000,
-		RndSeed:       time.Now().UnixNano(),
-		PostageAmount: 1000000,
-		PostageDepth:  20,
-		TxOnErrWait:   10 * time.Second,
-		RxOnErrWait:   10 * time.Second,
-		NodesSyncWait: time.Second * 30,
-		Duration:      12 * time.Hour,
-		GasPrice:      "100000000000",
-		MaxUseBatch:   time.Hour * 12,
+		ContentSize:     5000000,
+		RndSeed:         time.Now().UnixNano(),
+		PostageAmount:   1000000,
+		PostageDepth:    20,
+		TxOnErrWait:     10 * time.Second,
+		RxOnErrWait:     10 * time.Second,
+		NodesSyncWait:   time.Second * 30,
+		Duration:        12 * time.Hour,
+		UploadTimeout:   5 * time.Minute,
+		DownloadTimeout: 5 * time.Minute,
+		GasPrice:        "100000000000",
+		MaxUseBatch:     12 * time.Hour,
 	}
 }
 
@@ -89,7 +93,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 
 	batches := NewStore(o.MaxUseBatch)
 
-	test := &test{opt: o, ctx: ctx, clients: clients, logger: c.logger}
+	test := &test{clients: clients, logger: c.logger}
 
 	for i := 0; true; i++ {
 		select {
@@ -129,7 +133,10 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 			continue
 		}
 
+		var txCancel context.CancelFunc = func() {}
 		for retries := 0; retries < 3; retries++ {
+			txCancel()
+
 			select {
 			case <-ctx.Done():
 				return nil
@@ -138,6 +145,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 
 			c.metrics.UploadAttempts.Inc()
 
+			ctx, txCancel = context.WithTimeout(ctx, o.UploadTimeout)
+
 			batchID := batches.Get(txName)
 			if batchID == "" {
 				batchID, err = clients[txName].CreatePostageBatch(ctx, o.PostageAmount, o.PostageDepth, o.GasPrice, "load-test", true)
@@ -148,7 +157,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 				batches.Store(txName, batchID)
 			}
 
-			address, txDuration, err = test.uploadWithBatch(txName, txData, batchID)
+			address, txDuration, err = test.upload(ctx, txName, txData, batchID)
 			if err != nil {
 				c.metrics.UploadErrors.Inc()
 				c.logger.Infof("upload failed: %v", err)
@@ -157,6 +166,7 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 				break
 			}
 		}
+		txCancel()
 
 		if err != nil {
 			continue // skip
@@ -166,7 +176,10 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 
 		time.Sleep(o.NodesSyncWait)
 
+		var rxCancel context.CancelFunc = func() {}
 		for retries := 0; retries < 3; retries++ {
+			rxCancel()
+
 			select {
 			case <-ctx.Done():
 				return nil
@@ -175,7 +188,8 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 
 			c.metrics.DownloadAttempts.Inc()
 
-			rxData, rxDuration, err = test.download(rxName, address)
+			ctx, rxCancel = context.WithTimeout(ctx, o.DownloadTimeout)
+			rxData, rxDuration, err = test.download(ctx, rxName, address)
 			if err != nil {
 				c.metrics.DownloadErrors.Inc()
 				c.logger.Infof("download failed: %v", err)
@@ -210,23 +224,22 @@ func (c *Check) Run(ctx context.Context, cluster orchestration.Cluster, opts int
 			}
 			c.logger.Infof("data mismatch: found %d different bytes, ~%.2f%%", diff, float64(diff)/float64(txLen)*100)
 		}
+		rxCancel()
 	}
 
 	return nil
 }
 
 type test struct {
-	opt     Options
-	ctx     context.Context
 	clients map[string]*bee.Client
 	logger  logging.Logger
 }
 
-func (t *test) uploadWithBatch(cName string, data []byte, batchID string) (swarm.Address, time.Duration, error) {
+func (t *test) upload(ctx context.Context, cName string, data []byte, batchID string) (swarm.Address, time.Duration, error) {
 	client := t.clients[cName]
 	t.logger.Infof("node %s: uploading data, batch id %s", cName, batchID)
 	start := time.Now()
-	addr, err := client.UploadBytes(t.ctx, data, api.UploadOptions{Pin: false, BatchID: batchID, Direct: true})
+	addr, err := client.UploadBytes(ctx, data, api.UploadOptions{Pin: false, BatchID: batchID, Direct: true})
 	if err != nil {
 		return swarm.ZeroAddress, 0, fmt.Errorf("upload to the node %s: %w", cName, err)
 	}
@@ -236,11 +249,11 @@ func (t *test) uploadWithBatch(cName string, data []byte, batchID string) (swarm
 	return addr, txDuration, nil
 }
 
-func (t *test) download(cName string, addr swarm.Address) ([]byte, time.Duration, error) {
+func (t *test) download(ctx context.Context, cName string, addr swarm.Address) ([]byte, time.Duration, error) {
 	client := t.clients[cName]
 	t.logger.Infof("node %s: downloading address %s", cName, addr)
 	start := time.Now()
-	data, err := client.DownloadBytes(t.ctx, addr)
+	data, err := client.DownloadBytes(ctx, addr)
 	if err != nil {
 		return nil, 0, fmt.Errorf("download from node %s: %w", cName, err)
 	}
@@ -249,21 +262,3 @@ func (t *test) download(cName string, addr swarm.Address) ([]byte, time.Duration
 
 	return data, rxDuration, nil
 }
-
-// func (t *test) upload(cName string, data []byte) (swarm.Address, time.Duration, error) {
-// 	client := t.clients[cName]
-// 	batchID, err := client.GetOrCreateBatch(t.ctx, t.opt.PostageAmount, t.opt.PostageDepth, t.opt.GasPrice, "smoke-test")
-// 	if err != nil {
-// 		return swarm.ZeroAddress, 0, fmt.Errorf("node %s: unable to create batch id: %w", cName, err)
-// 	}
-// 	t.logger.Infof("node %s: uploading data, batch id %s", cName, batchID)
-// 	start := time.Now()
-// 	addr, err := client.UploadBytes(t.ctx, data, api.UploadOptions{Pin: false, BatchID: batchID, Direct: true})
-// 	if err != nil {
-// 		return swarm.ZeroAddress, 0, fmt.Errorf("upload to the node %s: %w", cName, err)
-// 	}
-// 	txDuration := time.Since(start)
-// 	t.logger.Infof("node %s: upload done in %s", cName, txDuration)
-
-// 	return addr, txDuration, nil
-// }