- Go update to 1.19

- Deps versions bump
- Code cleanup and refactoring

.gitignore

@@ -8,4 +8,6 @@ configuration/*/*
 !configuration/defaults/*
 c.out
 coverage.html
-.golangci.yml
+.golangci_*.yml
+linter.mk
+tests.mk

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.17-buster as build-env
+FROM golang:1.19-buster as build-env
 
 WORKDIR /app
 COPY . /app

Makefile

@@ -1,5 +1,5 @@
 IMAGE_NAME = spacetabio/prerender-go
-IMAGE_VERSION = v1.1.2
+IMAGE_VERSION = v1.2.0
 
 deps:
 	go mod vendor
@@ -54,28 +54,35 @@ run_headless_shell:
 
 ## -------------------
 
-## lint and test stuff
+# ----
+## LINTER stuff start
 
-get_lint_config:
-	@[ -f ./.golangci.yml ] && echo ".golangci.yml exists" || ( echo "getting .golangci.yml" && curl -O https://raw.githubusercontent.com/spacetab-io/docker-images-golang/master/linter/.golangci.yml )
-.PHONY: get_lint_config
+linter_include_check:
+	@[ -f linter.mk ] && echo "linter.mk include exists" || (echo "getting linter.mk from github.com" && curl -sO https://raw.githubusercontent.com/spacetab-io/makefiles/master/golang/linter.mk)
 
-lint: get_lint_config
-	golangci-lint run
 .PHONY: lint
+lint: linter_include_check
+	@make -f linter.mk go_lint
 
-test-unit:
-	go test ./... --race --cover -count=1 -timeout 1s -coverprofile=c.out -v
-.PHONY: test-unit
+## LINTER stuff end
+# ----
 
-coverage-html:
-	go tool cover -html=c.out -o coverage.html
-.PHONE: coverage-html
+# ----
+## TESTS stuff start
 
-test: deps test-unit coverage-html
-.PHONY: test
+tests_include_check:
+	@[ -f tests.mk ] && echo "tests.mk include exists" || (echo "getting tests.mk from github.com" && curl -sO https://raw.githubusercontent.com/spacetab-io/makefiles/master/golang/tests.mk)
 
-## -------------------
+tests: tests_include_check
+	@make -f tests.mk go_tests
+.PHONY: tests
+
+tests_html: tests_include_check
+	@make -f tests.mk go_tests_html
+.PHONY: tests_html
+
+## TESTS stuff end
+# ----
 
 ## image stuff
 

cmd/main.go

@@ -67,6 +67,7 @@ func getConfigs(ctx context.Context) (
 	return cfg, nil
 }
 
+//nolint:ireturn,nolintlint // we need it here
 func initCfgAndService(cmd *cobra.Command) (*configuration.Config, service.Service, error) {
 	cfg, err := getConfigs(cmd.Context())
 	if err != nil {

cmd/run.go

@@ -1,6 +1,7 @@
 package cmd
 
 import (
+	"fmt"
 	"runtime"
 	"time"
 
@@ -28,31 +29,30 @@ func run(cmd *cobra.Command, _ []string) error {
 		Str("lookup strategy", cfg.Prerender.Lookup.Type).
 		Str("render wait strategy", cfg.Prerender.WaitFor).
 		Msg("start rendering pages")
+
 	pages, err := srv.PreparePages(links)
 	if err != nil {
 		log.Error().Err(err).Msg("prepare pages error")
 
-		return err
+		return fmt.Errorf("prepare pages error: %w", err)
 	}
 
 	maxWorkers := countMaxWorkers(cfg)
 
-	if err := srv.RenderPages(pages, maxWorkers); err != nil {
+	if err := srv.RenderPages(cmd.Context(), pages, maxWorkers); err != nil {
 		log.Error().Err(err).Msg("render pages error")
 
-		return err
+		return fmt.Errorf("render pages error: %w", err)
 	}
 
-	timeEnd := time.Now()
-
-	srv.PrepareRenderReport(pages, timeEnd.Sub(timeStart), maxWorkers)
+	cmd.Print(srv.PrepareRenderReport(pages, time.Since(timeStart), maxWorkers))
 
 	return nil
 }
 
 func countMaxWorkers(cfg *configuration.Config) int {
 	numprocs := runtime.GOMAXPROCS(runtime.NumCPU())
-	maxWorkers := 2 * numprocs // nolint:gomnd
+	maxWorkers := 2 * numprocs //nolint:gomnd
 
 	if cfg.Prerender.ConcurrentLimit == 0 {
 		return numprocs

configuration/defaults/info.yaml

@@ -4,7 +4,7 @@ defaults:
     alias: "prerender"
     name: "SPA prerender"
     about: "Приложение для рендера SPA приложений как html странички"
-    version: "1.0.1"
+    version: "1.2.0"
     docs: "---"
     contacts: "[email protected]"
     copyright: "SpaceTab © 2022"

configuration/defaults/prerender.yaml

@@ -21,7 +21,7 @@ defaults: # stage name
         value: ""
     max_attempts: 5
     sleep_time: 3s
-    wait_timeout: 30s
+    wait_timeout: 1m
     render_period: 7h
     viewport:
       width: 1680

configuration/prerender.go

@@ -1,3 +1,4 @@
+//nolint:tagliatelle // legacy
 package configuration
 
 import (
@@ -27,17 +28,6 @@ type lookupConfig struct {
 	ParamsToSave []string `yaml:"get_params_to_save"`
 }
 
-func (c lookupConfig) GetSourceURL() string {
-	switch c.Type {
-	case models.LookupTypeSitemaps:
-		return strings.Join(c.SitemapURLs, ",")
-	case models.LookupTypeURLs:
-		return strings.Join(c.PageURLs, ", ")
-	}
-
-	return ""
-}
-
 type viewportConfig struct {
 	Width  int64 `yaml:"width"`
 	Height int64 `yaml:"height"`
@@ -59,6 +49,17 @@ type PrerenderConfig struct {
 	Page404Text     string         `yaml:"page_404_text"`
 }
 
+func (c lookupConfig) GetSourceURL() string {
+	switch c.Type {
+	case models.LookupTypeSitemaps:
+		return strings.Join(c.SitemapURLs, ",")
+	case models.LookupTypeURLs:
+		return strings.Join(c.PageURLs, ", ")
+	}
+
+	return ""
+}
+
 func (ec ElementConfig) GetWaitElement() string {
 	elem := ec.Type
 
@@ -76,8 +77,12 @@ func (ec ElementConfig) GetWaitElement() string {
 func (ec ElementConfig) GetWaitElementAttr(attrValue string) string {
 	elem := ec.GetWaitElement()
 
+	if ec.Attribute.Value == "" {
+		ec.Attribute.Value = attrValue
+	}
+
 	if ec.Attribute.Name != "" && ec.Attribute.Value != "" {
-		elem += fmt.Sprintf("[%s=%s]", ec.Attribute.Name, attrValue)
+		elem += fmt.Sprintf("[%s=%s]", ec.Attribute.Name, ec.Attribute.Value)
 	}
 
 	return elem

configuration/storage.go

@@ -27,7 +27,7 @@ type S3Config struct {
 }
 
 type LocalStorageConfig struct {
-	StoragePath string `yaml:"storage_path"`
+	StoragePath string `yaml:"storage_path"` //nolint:tagliatelle // legacy
 }
 
 type StorageConfig struct {

go.mod

@@ -1,67 +1,66 @@
 module github.com/spacetab-io/prerender-go
 
-go 1.17
+go 1.19
 
 require (
-	github.com/aws/aws-sdk-go-v2 v1.16.15
-	github.com/aws/aws-sdk-go-v2/config v1.17.6
-	github.com/aws/aws-sdk-go-v2/credentials v1.12.19
-	github.com/aws/aws-sdk-go-v2/service/s3 v1.27.10
-	github.com/chromedp/cdproto v0.0.0-20200709115526-d1f6fc58448b
-	github.com/chromedp/chromedp v0.5.3
-	github.com/spacetab-io/commands-go v0.5.0
-	github.com/spacetab-io/configuration-go v1.2.0
+	github.com/aws/aws-sdk-go-v2 v1.17.2
+	github.com/aws/aws-sdk-go-v2/config v1.18.4
+	github.com/aws/aws-sdk-go-v2/credentials v1.13.4
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.29.5
+	github.com/chromedp/cdproto v0.0.0-20220924210414-0e3390be1777
+	github.com/chromedp/chromedp v0.8.6
+	github.com/spacetab-io/commands-go v0.6.0
+	github.com/spacetab-io/configuration-go v1.2.1
 	github.com/spacetab-io/configuration-structs-go/v2 v2.0.0-alpha4
 	github.com/spacetab-io/logs-go/v3 v3.0.0-alpha3
-	github.com/spf13/cobra v1.5.0
-	github.com/stretchr/testify v1.8.0
+	github.com/spf13/cobra v1.6.1
+	github.com/stretchr/testify v1.8.1
 	github.com/yterajima/go-sitemap v0.3.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
 	github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
-	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.8 // indirect
-	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.22 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.16 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.23 // indirect
-	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.13 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.9 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.17 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.16 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sso v1.11.22 // indirect
-	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.4 // indirect
-	github.com/aws/aws-sdk-go-v2/service/sts v1.16.18 // indirect
-	github.com/aws/smithy-go v1.13.3 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.10 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.26 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.4.20 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.3.27 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.0.17 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.9.11 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.1.21 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.9.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.20 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.11.26 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.9 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.17.6 // indirect
+	github.com/aws/smithy-go v1.13.5 // indirect
+	github.com/chromedp/sysutil v1.0.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/getsentry/sentry-go v0.13.0 // indirect
-	github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee // indirect
-	github.com/gobwas/pool v0.2.0 // indirect
-	github.com/gobwas/ws v1.0.3 // indirect
+	github.com/gobwas/httphead v0.1.0 // indirect
+	github.com/gobwas/pool v0.2.1 // indirect
+	github.com/gobwas/ws v1.1.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/jackc/chunkreader/v2 v2.0.1 // indirect
-	github.com/jackc/pgconn v1.10.0 // indirect
-	github.com/jackc/pgio v1.0.0 // indirect
+	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/jackc/pgpassfile v1.0.0 // indirect
-	github.com/jackc/pgproto3/v2 v2.1.1 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
-	github.com/jackc/pgtype v1.8.1 // indirect
-	github.com/jackc/pgx/v4 v4.13.0 // indirect
-	github.com/knq/sysutil v0.0.0-20191005231841-15668db23d08 // indirect
-	github.com/mailru/easyjson v0.7.1 // indirect
+	github.com/jackc/pgx-zap v0.0.0-20220909013905-c08a18c611dd // indirect
+	github.com/jackc/pgx/v5 v5.0.4 // indirect
+	github.com/josharian/intern v1.0.0 // indirect
+	github.com/kr/text v0.1.0 // indirect
+	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/pressly/goose v2.7.0+incompatible // indirect
+	github.com/rogpeppe/go-internal v1.9.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	go.uber.org/zap v1.21.0 // indirect
-	golang.org/x/crypto v0.0.0-20210921155107-089bfa567519 // indirect
-	golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac // indirect
-	golang.org/x/text v0.3.7 // indirect
+	go.uber.org/zap v1.23.0 // indirect
+	golang.org/x/crypto v0.0.0-20220829220503-c86fa9a7ed90 // indirect
+	golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect
+	golang.org/x/text v0.3.8 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )