Merging the latest updates from develop

.codespellrc

@@ -0,0 +1,7 @@
+[codespell]
+# js/ folder seems to contain vendored code
+skip = .git,*.pdf,*.svg,*.min.js,*.js.map,js,requirements.txt
+# docs/videos/healthcare-and-life-sciences/lysozyme-example/submit.sh has CONECT - might be "legit"
+ignore-regex = HETATM -e CONECT|\([A-Z]\)[a-z]+
+#
+ignore-words-list = namd,tempdate,te,ue,startd,passtime

.gitignore

@@ -63,6 +63,9 @@ packer-manifest.json
 #### Python Virtual Environments
 venv*/
 
+### Python cache files
+**/__pycache__
+
 #### Exclude from gitingore
 !tools/validate_configs/golden_copies/expectations/*/*/*/defaults.auto.pkrvars.hcl
 !tools/validate_configs/golden_copies/expectations/*/*/terraform.tfvars

.pre-commit-config.yaml

@@ -39,6 +39,7 @@ repos:
     entry: tools/autodoc/terraform_docs.sh
     language: script
     files: ^.*\.pkr\.hcl$
+    exclude: (pkg\/.*$)|(tools/validate_configs/golden_copies/.*)
     pass_filenames: true
     require_serial: true
   - id: duplicate-diff
@@ -57,6 +58,14 @@ repos:
     files: '.*\.tf$'
     pass_filenames: false
     require_serial: true
+  - id: addlicense
+    name: addlicense
+    language: system
+    entry: addlicense
+    args: ['-c', '"Google LLC"', '-l', 'apache']
+    exclude: docs/videos/healthcare-and-life-sciences/lysozyme-example/submit.sh
+    pass_filenames: true
+
 - repo: https://github.com/dnephin/pre-commit-golang
   rev: v0.5.1
   hooks:
@@ -66,13 +75,12 @@ repos:
   - id: go-cyclo
     args: [-over=15]
   - id: go-unit-tests
-  - id: go-build
   - id: go-mod-tidy
-- repo: https://github.com/tekwizely/pre-commit-golang
-  rev: v1.0.0-rc.1
-  hooks:
-  - id: go-critic
-    args: [-disable, "#experimental,sloppyTypeAssert"]
+# Disabled temporarily due to https://github.com/go-critic/go-critic/issues/1388
+# - repo: https://github.com/tekwizely/pre-commit-golang
+#   rev: v1.0.0-rc.1
+#   hooks:
+#   - id: go-critic
 - repo: https://github.com/Bahjat/pre-commit-golang
   rev: v1.0.2
   hooks:
@@ -105,4 +113,9 @@ repos:
   rev: v4.4.0
   hooks:
   - id: end-of-file-fixer
+- repo: https://github.com/codespell-project/codespell
+  rev: v2.2.6
+  hooks:
+  - id: codespell
+    exclude: requirements.txt$|/js/
 exclude: tools/validate_configs/golden_copies/.*

.tflint.hcl

@@ -11,11 +11,17 @@
 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 // See the License for the specific language governing permissions and
 // limitations under the License.
+
 plugin "google" {
   enabled = true
   version = "0.23.0"
   source  = "github.com/terraform-linters/tflint-ruleset-google"
 }
+plugin "terraform" {
+  enabled = true
+  version = "0.4.0"
+  source  = "github.com/terraform-linters/tflint-ruleset-terraform"
+}
 rule "terraform_deprecated_index" {
   enabled = true
 }

Makefile

@@ -3,7 +3,7 @@ MIN_PACKER_VERSION=1.7.9 # for building images
 MIN_TERRAFORM_VERSION=1.2 # for deploying modules
 MIN_GOLANG_VERSION=1.18 # for building ghpc
 
-.PHONY: install install-user tests format add-google-license install-dev-deps \
+.PHONY: install install-user tests format install-dev-deps \
         warn-go-missing warn-terraform-missing warn-packer-missing \
         warn-go-version warn-terraform-version warn-packer-version \
         test-engine validate_configs validate_golden_copy packer-check \
@@ -65,15 +65,6 @@ install-dev-deps: warn-terraform-version warn-packer-version check-pre-commit ch
 	go install golang.org/x/tools/cmd/goimports@latest
 	go install honnef.co/go/tools/cmd/staticcheck@latest
 
-ifeq (, $(shell which addlicense))
-add-google-license:
-	$(error "could not find addlicense in PATH, run: go install github.com/google/addlicense@latest")
-else
-add-google-license:
-	# lysozyme-example is under CC-BY-4.0
-	addlicense -c "Google LLC" -l apache -ignore **/lysozyme-example/submit.sh .
-endif
-
 # RULES SUPPORTING THE ABOVE
 
 test-engine: warn-go-missing

cmd/create.go

@@ -21,9 +21,9 @@ import (
 	"errors"
 	"fmt"
 	"hpc-toolkit/pkg/config"
+	"hpc-toolkit/pkg/logging"
 	"hpc-toolkit/pkg/modulewriter"
 	"hpc-toolkit/pkg/validators"
-	"log"
 	"path/filepath"
 	"strings"
 
@@ -83,46 +83,43 @@ func runCreateCmd(cmd *cobra.Command, args []string) {
 	deplDir := filepath.Join(outputDir, deplName)
 	checkErr(modulewriter.WriteDeployment(dc, deplDir, overwriteDeployment))
 
-	fmt.Println("To deploy your infrastructure please run:")
-	fmt.Println()
-	fmt.Printf(boldGreen("%s deploy %s\n"), execPath(), deplDir)
-	fmt.Println()
+	logging.Info("To deploy your infrastructure please run:")
+	logging.Info("")
+	logging.Info(boldGreen("%s deploy %s"), execPath(), deplDir)
+	logging.Info("")
 	printAdvancedInstructionsMessage(deplDir)
 }
 
 func printAdvancedInstructionsMessage(deplDir string) {
-	fmt.Println("Find instructions for cleanly destroying infrastructure and advanced manual")
-	fmt.Println("deployment instructions at:")
-	fmt.Println()
-	fmt.Printf("%s\n", modulewriter.InstructionsPath(deplDir))
+	logging.Info("Find instructions for cleanly destroying infrastructure and advanced manual")
+	logging.Info("deployment instructions at:")
+	logging.Info("")
+	logging.Info(modulewriter.InstructionsPath(deplDir))
 }
 
 func expandOrDie(path string) config.DeploymentConfig {
 	dc, ctx, err := config.NewDeploymentConfig(path)
 	if err != nil {
-		log.Fatal(renderError(err, ctx))
+		logging.Fatal(renderError(err, ctx))
 	}
 	// Set properties from CLI
 	if err := setCLIVariables(&dc.Config, cliVariables); err != nil {
-		log.Fatalf("Failed to set the variables at CLI: %v", err)
+		logging.Fatal("Failed to set the variables at CLI: %v", err)
 	}
 	if err := setBackendConfig(&dc.Config, cliBEConfigVars); err != nil {
-		log.Fatalf("Failed to set the backend config at CLI: %v", err)
-	}
-	if err := setValidationLevel(&dc.Config, validationLevel); err != nil {
-		log.Fatal(err)
-	}
-	if err := skipValidators(&dc); err != nil {
-		log.Fatal(err)
+		logging.Fatal("Failed to set the backend config at CLI: %v", err)
 	}
+	checkErr(setValidationLevel(&dc.Config, validationLevel))
+	checkErr(skipValidators(&dc))
+
 	if dc.Config.GhpcVersion != "" {
-		fmt.Printf("ghpc_version setting is ignored.")
+		logging.Info("ghpc_version setting is ignored.")
 	}
 	dc.Config.GhpcVersion = GitCommitInfo
 
 	// Expand the blueprint
 	if err := dc.ExpandConfig(); err != nil {
-		log.Fatal(renderError(err, ctx))
+		logging.Fatal(renderError(err, ctx))
 	}
 
 	validateMaybeDie(dc.Config, ctx)
@@ -134,29 +131,29 @@ func validateMaybeDie(bp config.Blueprint, ctx config.YamlCtx) {
 	if err == nil {
 		return
 	}
-	log.Println(renderError(err, ctx))
-
-	log.Println("One or more blueprint validators has failed. See messages above for suggested")
-	log.Println("actions. General troubleshooting guidance and instructions for configuring")
-	log.Println("validators are shown below.")
-	log.Println("")
-	log.Println("- https://goo.gle/hpc-toolkit-troubleshooting")
-	log.Println("- https://goo.gle/hpc-toolkit-validation")
-	log.Println("")
-	log.Println("Validators can be silenced or treated as warnings or errors:")
-	log.Println("")
-	log.Println("- https://goo.gle/hpc-toolkit-validation-levels")
-	log.Println("")
+	logging.Error(renderError(err, ctx))
+
+	logging.Error("One or more blueprint validators has failed. See messages above for suggested")
+	logging.Error("actions. General troubleshooting guidance and instructions for configuring")
+	logging.Error("validators are shown below.")
+	logging.Error("")
+	logging.Error("- https://goo.gle/hpc-toolkit-troubleshooting")
+	logging.Error("- https://goo.gle/hpc-toolkit-validation")
+	logging.Error("")
+	logging.Error("Validators can be silenced or treated as warnings or errors:")
+	logging.Error("")
+	logging.Error("- https://goo.gle/hpc-toolkit-validation-levels")
+	logging.Error("")
 
 	switch bp.ValidationLevel {
 	case config.ValidationWarning:
 		{
-			log.Println(boldYellow("Validation failures were treated as a warning, continuing to create blueprint."))
-			log.Println("")
+			logging.Error(boldYellow("Validation failures were treated as a warning, continuing to create blueprint."))
+			logging.Error("")
 		}
 	case config.ValidationError:
 		{
-			log.Fatal(boldRed("validation failed due to the issues listed above"))
+			logging.Fatal(boldRed("validation failed due to the issues listed above"))
 		}
 	}
 

cmd/deploy.go

@@ -18,9 +18,9 @@ package cmd
 import (
 	"fmt"
 	"hpc-toolkit/pkg/config"
+	"hpc-toolkit/pkg/logging"
 	"hpc-toolkit/pkg/modulewriter"
 	"hpc-toolkit/pkg/shell"
-	"log"
 	"path/filepath"
 
 	"github.com/spf13/cobra"
@@ -98,7 +98,7 @@ func runDeployCmd(cmd *cobra.Command, args []string) {
 			checkErr(fmt.Errorf("group %s is an unsupported kind %s", groupDir, group.Kind().String()))
 		}
 	}
-	fmt.Println("\n###############################")
+	logging.Info("\n###############################")
 	printAdvancedInstructionsMessage(deploymentRoot)
 }
 
@@ -131,15 +131,15 @@ func deployPackerGroup(moduleDir string) error {
 	}
 	buildImage := applyBehavior == shell.AutomaticApply || shell.ApplyChangesChoice(c)
 	if buildImage {
-		log.Printf("initializing packer module at %s", moduleDir)
+		logging.Info("initializing packer module at %s", moduleDir)
 		if err := shell.ExecPackerCmd(moduleDir, false, "init", "."); err != nil {
 			return err
 		}
-		log.Printf("validating packer module at %s", moduleDir)
+		logging.Info("validating packer module at %s", moduleDir)
 		if err := shell.ExecPackerCmd(moduleDir, false, "validate", "."); err != nil {
 			return err
 		}
-		log.Printf("building image using packer module at %s", moduleDir)
+		logging.Info("building image using packer module at %s", moduleDir)
 		if err := shell.ExecPackerCmd(moduleDir, true, "build", "."); err != nil {
 			return err
 		}

cmd/expand.go

@@ -16,7 +16,7 @@
 package cmd
 
 import (
-	"fmt"
+	"hpc-toolkit/pkg/logging"
 
 	"github.com/spf13/cobra"
 )
@@ -50,5 +50,5 @@ var (
 func runExpandCmd(cmd *cobra.Command, args []string) {
 	dc := expandOrDie(args[0])
 	checkErr(dc.ExportBlueprint(outputFilename))
-	fmt.Printf(boldGreen("Expanded Environment Definition created successfully, saved as %s.\n"), outputFilename)
+	logging.Info(boldGreen("Expanded Environment Definition created successfully, saved as %s."), outputFilename)
 }

cmd/export_test.go

@@ -18,40 +18,24 @@ package cmd
 
 import (
 	"os"
+	"path/filepath"
 
 	. "gopkg.in/check.v1"
 )
 
 func (s *MySuite) TestIsDir(c *C) {
-	dir, err := os.MkdirTemp("", "test-*")
-	if err != nil {
-		c.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
+	dir := c.MkDir()
+	c.Assert(checkDir(nil, []string{dir}), IsNil)
 
-	err = checkDir(nil, []string{dir})
-	c.Assert(err, IsNil)
+	p := filepath.Join(dir, "does-not-exist")
+	c.Assert(checkDir(nil, []string{p}), NotNil)
 
-	os.RemoveAll(dir)
-	err = checkDir(nil, []string{dir})
-	c.Assert(err, NotNil)
-
-	f, err := os.CreateTemp("", "test-*")
-	if err != nil {
-		c.Fatal(err)
-	}
-	defer os.Remove(f.Name())
-	err = checkDir(nil, []string{f.Name()})
-	c.Assert(err, NotNil)
+	f, err := os.CreateTemp(dir, "test-*")
+	c.Assert(err, IsNil)
+	c.Assert(checkDir(nil, []string{f.Name()}), NotNil)
 }
 
 func (s *MySuite) TestRunExport(c *C) {
-	dir, err := os.MkdirTemp("", "test-*")
-	if err != nil {
-		c.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-
-	err = runExportCmd(nil, []string{dir})
-	c.Assert(err, NotNil)
+	dir := c.MkDir()
+	c.Assert(runExportCmd(nil, []string{dir}), NotNil)
 }

cmd/root.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"hpc-toolkit/pkg/config"
-	"log"
+	"hpc-toolkit/pkg/logging"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -49,7 +49,7 @@ var (
 HPC deployments on the Google Cloud Platform.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			if err := cmd.Help(); err != nil {
-				log.Fatalf("cmd.Help function failed: %s", err)
+				logging.Fatal("cmd.Help function failed: %s", err)
 			}
 		},
 		Version:     "v1.26.0",
@@ -66,12 +66,9 @@ func init() {
 
 // Execute the root command
 func Execute() error {
-	// Don't prefix messages with data & time to improve readability.
-	// See https://pkg.go.dev/log#pkg-constants
-	log.SetFlags(0)
 	mismatch, branch, hash, dir := checkGitHashMismatch()
 	if mismatch {
-		fmt.Fprintf(os.Stderr, "WARNING: ghpc binary was built from a different commit (%s/%s) than the current git branch in %s (%s/%s). You can rebuild the binary by running 'make'\n",
+		logging.Error("WARNING: ghpc binary was built from a different commit (%s/%s) than the current git branch in %s (%s/%s). You can rebuild the binary by running 'make'",
 			GitBranch, GitCommitHash[0:7], dir, branch, hash[0:7])
 	}
 
@@ -198,7 +195,7 @@ func execPath() string {
 		// "simplification" of `ghpc` to `./ghpc`
 		return nice
 	}
-	// Code bellow assumes that `args0` contains path to file, not a
+	// Code below assumes that `args0` contains path to file, not a
 	// executable name from PATH.
 
 	{ // Find shortest & nicest form of args0
@@ -249,11 +246,11 @@ func execPath() string {
 	return args0
 }
 
-// checkErr is similar to cobra.CheckErr, but with renderError and log.Fatal
+// checkErr is similar to cobra.CheckErr, but with renderError and logging.Fatal
 // NOTE: this function uses empty YamlCtx, so if you have one, use renderError directly.
 func checkErr(err error) {
 	if err != nil {
 		msg := fmt.Sprintf("%s: %s", boldRed("Error"), renderError(err, config.YamlCtx{}))
-		log.Fatal(msg)
+		logging.Fatal(msg)
 	}
 }