Merge pull request GoogleCloudPlatform#1762 from GoogleCloudPlatform/…

…release-candidate

Release v1.23.0

.github/dependabot.yml

@@ -21,7 +21,7 @@ updates:
   labels:
   - dependencies
   - go
-  - chore
+  - release-chore
   schedule:
     interval: weekly
     day: monday

.github/release.yml

@@ -22,23 +22,18 @@ changelog:
     authors: []
   categories:
   - title: Key New Features 🎉
-    labels:
-    - release-key-new-features
+    labels: [release-key-new-features]
   - title: New Modules 🧱
-    labels:
-    - release-new-modules
-  - title: Module Improvements 🛠
-    labels:
-    - release-module-improvements
-  - title: Improvements
-    labels:
-    - release-improvements
-  - title: Deprecations
-    labels:
-    - release-deprecations
-  - title: Version Updates
-    labels:
-    - release-version-updates
+    labels: [release-new-modules]
+  - title: Module Improvements 🔨
+    labels: [release-module-improvements]
+  - title: Improvements 🛠
+    labels: [release-improvements]
+  - title: Deprecations 💤
+    labels: [release-deprecations]
+  - title: Version Updates ⏫
+    labels: [release-version-updates]
+  - title: Bug fixes 🐞
+    labels: [release-bugfix]
   - title: Other changes
-    labels:
-    - "*"
+    labels: ["*"]

.github/workflows/close-inactive-issues.yml

@@ -0,0 +1,36 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+name: Close inactive issues
+on:
+  schedule:
+  - cron: "30 1 * * *"
+
+jobs:
+  close-issues:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+    - uses: actions/stale@v5
+      with:
+        days-before-issue-stale: 30
+        days-before-issue-close: 14
+        stale-issue-label: "stale"
+        stale-issue-message: "This issue is stale because it has been open for 30 days with no activity."
+        close-issue-message: "This issue was closed because it has been inactive for 14 days since being marked as stale."
+        days-before-pr-stale: -1
+        days-before-pr-close: -1
+        repo-token: ${{ secrets.GITHUB_TOKEN }}

.pre-commit-config.yaml

@@ -15,7 +15,7 @@
 ---
 repos:
 - repo: https://github.com/antonbabenko/pre-commit-terraform
-  rev: v1.77.1
+  rev: v1.82.0
   hooks:
   - id: terraform_fmt
   - id: terraform_tflint
@@ -75,12 +75,12 @@ repos:
   - id: go-critic
     args: [-disable, "#experimental,sloppyTypeAssert"]
 - repo: https://github.com/adrienverge/yamllint
-  rev: v1.29.0
+  rev: v1.32.0
   hooks:
   - id: yamllint
     args: [-c=.yamllint]
 - repo: https://github.com/jackdewinter/pymarkdown
-  rev: v0.9.9
+  rev: v0.9.12
   hooks:
   - id: pymarkdown
     # Rules at https://github.com/jackdewinter/pymarkdown/tree/main/docs/rules

Makefile

@@ -21,7 +21,7 @@ ifneq (,$(wildcard .git))
 ## GIT DIRECTORY EXISTS
 GIT_TAG_VERSION=$(shell git tag --points-at HEAD)
 GIT_BRANCH=$(shell git branch --show-current)
-GIT_COMMIT_INFO=$(shell git describe --tags --dirty --long)
+GIT_COMMIT_INFO=$(shell git describe --tags --dirty --long --always)
 GIT_COMMIT_HASH=$(shell git rev-parse HEAD)
 GIT_INITIAL_HASH=$(shell git rev-list --max-parents=0 HEAD)
 endif

cmd/create.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"hpc-toolkit/pkg/config"
 	"hpc-toolkit/pkg/modulewriter"
+	"hpc-toolkit/pkg/validators"
 	"log"
 	"path/filepath"
 	"strings"
@@ -124,9 +125,43 @@ func expandOrDie(path string) config.DeploymentConfig {
 		log.Fatal(renderError(err, ctx))
 	}
 
+	validateMaybeDie(dc.Config, ctx)
 	return dc
 }
 
+func validateMaybeDie(bp config.Blueprint, ctx config.YamlCtx) {
+	err := validators.Execute(bp)
+	if err == nil {
+		return
+	}
+	log.Println(renderError(err, ctx))
+
+	log.Println("One or more blueprint validators has failed. See messages above for suggested")
+	log.Println("actions. General troubleshooting guidance and instructions for configuring")
+	log.Println("validators are shown below.")
+	log.Println("")
+	log.Println("- https://goo.gle/hpc-toolkit-troubleshooting")
+	log.Println("- https://goo.gle/hpc-toolkit-validation")
+	log.Println("")
+	log.Println("Validators can be silenced or treated as warnings or errors:")
+	log.Println("")
+	log.Println("- https://goo.gle/hpc-toolkit-validation-levels")
+	log.Println("")
+
+	switch bp.ValidationLevel {
+	case config.ValidationWarning:
+		{
+			log.Println("Validation failures were treated as a warning, continuing to create blueprint.")
+			log.Println("")
+		}
+	case config.ValidationError:
+		{
+			log.Fatal("validation failed due to the issues listed above")
+		}
+	}
+
+}
+
 func findPos(path config.Path, ctx config.YamlCtx) (config.Pos, bool) {
 	pos, ok := ctx.Pos(path)
 	for !ok && path.Parent() != nil {
@@ -137,30 +172,30 @@ func findPos(path config.Path, ctx config.YamlCtx) (config.Pos, bool) {
 }
 
 func renderError(err error, ctx config.YamlCtx) string {
-	var me config.Errors
-	if errors.As(err, &me) {
+	switch te := err.(type) {
+	case config.Errors:
 		var sb strings.Builder
-		for _, e := range me.Errors {
+		for _, e := range te.Errors {
 			sb.WriteString(renderError(e, ctx))
 			sb.WriteString("\n")
 		}
 		return sb.String()
-	}
-
-	var be config.BpError
-	if errors.As(err, &be) {
-		if pos, ok := findPos(be.Path, ctx); ok {
-			return renderRichError(be.Err, pos, ctx)
+	case validators.ValidatorError:
+		return fmt.Sprintf("validator %q failed:\n%v\n", te.Validator, renderError(te.Err, ctx))
+	case config.BpError:
+		if pos, ok := findPos(te.Path, ctx); ok {
+			return renderRichError(te.Err, pos, ctx)
 		}
+		return renderError(te.Err, ctx)
+	default:
+		return err.Error()
 	}
-	return err.Error()
 }
 
 func renderRichError(err error, pos config.Pos, ctx config.YamlCtx) string {
 	pref := fmt.Sprintf("%d: ", pos.Line)
 	arrow := strings.Repeat(" ", len(pref)+pos.Column-1) + "^"
-	return fmt.Sprintf(`
-Error: %s
+	return fmt.Sprintf(`Error: %s
 %s%s
 %s`, err, pref, ctx.Lines[pos.Line-1], arrow)
 }

cmd/create_test.go

@@ -136,12 +136,12 @@ func (s *MySuite) TestRenderError(c *C) {
 		got := renderError(err, config.YamlCtx{})
 		c.Check(got, Equals, "arbuz")
 	}
-	{ // has pos, but context is missing
+	{ // has pos, but context doesn't contain it
 		ctx := config.NewYamlCtx([]byte(``))
 		pth := config.Root.Vars.Dot("kale")
 		err := config.BpError{Path: pth, Err: errors.New("arbuz")}
 		got := renderError(err, ctx)
-		c.Check(got, Equals, "vars.kale: arbuz")
+		c.Check(got, Equals, "arbuz")
 	}
 	{ // has pos, has context
 		ctx := config.NewYamlCtx([]byte(`
@@ -150,9 +150,16 @@ vars:
 		pth := config.Root.Vars.Dot("kale")
 		err := config.BpError{Path: pth, Err: errors.New("arbuz")}
 		got := renderError(err, ctx)
-		c.Check(got, Equals, `
-Error: arbuz
+		c.Check(got, Equals, `Error: arbuz
 3:   kale: dos
            ^`)
 	}
 }
+
+func (s *MySuite) TestValidateMaybeDie(c *C) {
+	bp := config.Blueprint{
+		Validators:      []config.Validator{{Validator: "invalid"}},
+		ValidationLevel: config.ValidationWarning,
+	}
+	validateMaybeDie(bp, config.NewYamlCtx([]byte{})) // smoke test
+}

cmd/root.go

@@ -50,7 +50,7 @@ HPC deployments on the Google Cloud Platform.`,
 				log.Fatalf("cmd.Help function failed: %s", err)
 			}
 		},
-		Version:     "v1.22.1",
+		Version:     "v1.23.0",
 		Annotations: annotation,
 	}
 )

community/examples/htc-htcondor.yaml

@@ -22,6 +22,7 @@ vars:
   zone: us-central1-c
   disk_size_gb: 100
   new_image_family: htcondor-10x
+  enable_shielded_vm: true
 
 # Documentation for each of the modules used below can be found at
 # https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md

community/examples/tutorial-starccm-slurm.yaml

@@ -0,0 +1,114 @@
+# Copyright 2023 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+---
+
+blueprint_name: starccm-on-slurm
+
+vars:
+  project_id:  ## Set GCP Project ID Here ##
+  deployment_name: starccm-slurm
+  region: us-central1
+  zone: us-central1-c
+
+# Documentation for each of the modules used below can be found at
+# https://github.com/GoogleCloudPlatform/hpc-toolkit/blob/main/modules/README.md
+
+deployment_groups:
+- group: primary
+  modules:
+  # Source is an embedded resource, denoted by "resources/*" without ./, ../, /
+  # as a prefix. To refer to a local resource, prefix with ./, ../ or /
+  # Example - ./resources/network/vpc
+  - id: network1
+    source: modules/network/vpc
+
+  - id: homefs
+    source: modules/file-system/filestore
+    use: [network1]
+    settings:
+      local_mount: /home
+
+  - id: login-script
+    kind: terraform
+    source: modules/scripts/startup-script
+    settings:
+      configure_ssh_host_patterns: ["star*"]
+
+  - id: compute-script
+    source: modules/scripts/startup-script
+    settings:
+      configure_ssh_host_patterns: ["star*"]
+      runners:
+      - type: shell
+        content: |
+          #!/bin/bash
+          google_mpi_tuning --hpcthroughput
+          google_mpi_tuning --nomitigation
+        destination: /tmp/tune-mpi.sh
+
+  - id: debug_node_group
+    source: community/modules/compute/schedmd-slurm-gcp-v5-node-group
+    settings:
+      node_count_dynamic_max: 4
+      machine_type: n2-standard-2
+
+  - id: debug_partition
+    source: community/modules/compute/schedmd-slurm-gcp-v5-partition
+    use:
+    - network1
+    - homefs
+    - debug_node_group
+    - compute-script
+    settings:
+      partition_name: debug
+      is_default: true
+
+  - id: compute_node_group
+    source: community/modules/compute/schedmd-slurm-gcp-v5-node-group
+    settings:
+      bandwidth_tier: "gvnic_enabled"
+      disable_public_ips: false
+      machine_type: c2-standard-60
+      node_count_dynamic_max: 20
+
+  - id: compute_partition
+    source: community/modules/compute/schedmd-slurm-gcp-v5-partition
+    use:
+    - network1
+    - homefs
+    - compute_node_group
+    - compute-script
+    settings:
+      partition_name: compute
+
+  - id: slurm_controller
+    source: community/modules/scheduler/schedmd-slurm-gcp-v5-controller
+    use:
+    - network1
+    - debug_partition
+    - compute_partition
+    - homefs
+    settings:
+      disable_controller_public_ips: true
+
+  - id: slurm_login
+    source: community/modules/scheduler/schedmd-slurm-gcp-v5-login
+    use:
+    - network1
+    - slurm_controller
+    - login-script
+    settings:
+      machine_type: n2-standard-4
+      disable_login_public_ips: true

community/examples/tutorial-starccm.yaml

@@ -29,6 +29,7 @@ deployment_groups:
     kind: terraform
     id: startup
     settings:
+      configure_ssh_host_patterns: ["star*"]
       runners:
       - type: shell
         content: |