From 83a4ef15dfda6403862437f8acf3606f255e888a Mon Sep 17 00:00:00 2001 From: Peeyush Gupta Date: Tue, 19 Nov 2019 08:58:58 -0500 Subject: [PATCH 01/18] Adding build for ppc64le --- build.make | 1 + 1 file changed, 1 insertion(+) diff --git a/build.make b/build.make index 1b6f35fe1c..7075a37eed 100644 --- a/build.make +++ b/build.make @@ -70,6 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* fi container-%: build-% From 003c14b2d4ae3b1463db5e5b3ff91f39b03f5ba8 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Mon, 11 Nov 2019 23:49:42 -0800 Subject: [PATCH 02/18] Add snapshotter CRDs after cluster setup Signed-off-by: Grant Griffiths --- prow.sh | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) diff --git a/prow.sh b/prow.sh index 3b9621a21a..bc9c9f03bd 100755 --- a/prow.sh +++ b/prow.sh @@ -322,6 +322,9 @@ configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha f configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" +# Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag" + # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply # doesn't work. Such tests should be written in a way that they verify @@ -657,6 +660,59 @@ install_hostpath () { fi } +# Installs all nessesary snapshotter CRDs +install_snapshot_crds() { + # Wait until volumesnapshot CRDs are in place. + CRD_BASE_DIR="https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/config/crd" + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotclasses.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshots.yaml" --validate=false + kubectl apply -f "${CRD_BASE_DIR}/snapshot.storage.k8s.io_volumesnapshotcontents.yaml" --validate=false + cnt=0 + until kubectl get volumesnapshotclasses.snapshot.storage.k8s.io \ + && kubectl get volumesnapshots.snapshot.storage.k8s.io \ + && kubectl get volumesnapshotcontents.snapshot.storage.k8s.io; do + if [ $cnt -gt 30 ]; then + echo >&2 "ERROR: snapshot CRDs not ready after over 1 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot CRDs, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 2 + done +} + +# Install snapshot controller and associated RBAC, retrying until the pod is running. +install_snapshot_controller() { + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml" + cnt=0 + until kubectl get clusterrolebinding snapshot-controller-role; do + if [ $cnt -gt 30 ]; then + echo "Cluster role bindings:" + kubectl describe clusterrolebinding + echo >&2 "ERROR: snapshot controller RBAC not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot RBAC setup complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done + + + kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" + cnt=0 + until kubectl get statefulset snapshot-controller | grep snapshot-controller | grep "1/1"; do + if [ $cnt -gt 30 ]; then + echo "Running statefulsets:" + kubectl describe statefulsets + echo >&2 "ERROR: snapshot controller not ready after over 5 min" + exit 1 + fi + echo "$(date +%H:%M:%S)" "waiting for snapshot controller deployment to complete, attempt #$cnt" + cnt=$((cnt + 1)) + sleep 10 + done +} + # collect logs and cluster status (like the version of all components, Kubernetes version, test version) collect_cluster_info () { cat < Date: Sat, 30 Nov 2019 00:29:00 +0530 Subject: [PATCH 03/18] Use kind v0.6.0 kind v0.6.0 appends the kubeconfig with the default config at ~/.kube/config. --- prow.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index d719e9808d..b882306297 100755 --- a/prow.sh +++ b/prow.sh @@ -107,8 +107,7 @@ configvar CSI_PROW_GO_VERSION_GINKGO "${CSI_PROW_GO_VERSION_BUILD}" "Go version # kind version to use. If the pre-installed version is different, # the desired version is downloaded from https://github.com/kubernetes-sigs/kind/releases/download/ # (if available), otherwise it is built from source. -# TODO: https://github.com/kubernetes-csi/csi-release-tools/issues/39 -configvar CSI_PROW_KIND_VERSION "86bc23d84ac12dcb56a0528890736e2c347c2dc3" "kind" +configvar CSI_PROW_KIND_VERSION "v0.6.0" "kind" # ginkgo test runner version to use. If the pre-installed version is # different, the desired version is built from source. @@ -579,8 +578,7 @@ EOF die "Cluster creation failed again, giving up. See the 'kind-cluster' artifact directory for additional logs." fi fi - KUBECONFIG="$(kind get kubeconfig-path --name=csi-prow)" - export KUBECONFIG + export KUBECONFIG="${HOME}/.kube/config" } # Deletes kind cluster inside a prow job From 9a7a685ee169d669a0182532315aeecd8a8715dc Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 2 Dec 2019 17:45:57 -0800 Subject: [PATCH 04/18] Create a kind cluster with two worker nodes so that the topology feature can be tested. Test cases that test accessing volumes from multiple nodes need to be skipped --- prow.sh | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index b882306297..815bc9d8a6 100755 --- a/prow.sh +++ b/prow.sh @@ -330,7 +330,11 @@ configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag # whether they can run with the current cluster provider, but until # they are, we filter them out by name. Like the other test selection # variables, this is again a space separated list of regular expressions. -configvar CSI_PROW_E2E_SKIP 'Disruptive' "tests that need to be skipped" +# +# "different node" test skips can be removed once +# https://github.com/kubernetes/kubernetes/pull/82678 has been backported +# to all the K8s versions we test against +configvar CSI_PROW_E2E_SKIP 'Disruptive|different\s+node' "tests that need to be skipped" # This is the directory for additional result files. Usually set by Prow, but # if not (for example, when invoking manually) it defaults to the work directory. @@ -526,6 +530,7 @@ apiVersion: kind.sigs.k8s.io/v1alpha3 nodes: - role: control-plane - role: worker +- role: worker EOF # kubeadm has API dependencies between apiVersion and Kubernetes version @@ -840,10 +845,6 @@ run_e2e () ( install_e2e || die "building e2e.test failed" install_ginkgo || die "installing ginkgo failed" - # TODO (?): multi-node cluster (depends on https://github.com/kubernetes-csi/csi-driver-host-path/pull/14). - # When running on a multi-node cluster, we need to figure out where the - # hostpath driver was deployed and set ClientNodeName accordingly. - generate_test_driver >"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again From 53888ae7d59b862bfded46b883570c98f5fc5fff Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Tue, 3 Dec 2019 18:18:38 -0800 Subject: [PATCH 05/18] Improve README by adding an explicit Kubernetes dependency section --- README.md | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index cc40f161cd..08c828846c 100644 --- a/README.md +++ b/README.md @@ -141,17 +141,6 @@ The `vendor` directory is optional. It is still present in projects because it avoids downloading sources during CI builds. If this is no longer deemed necessary, then a project can also remove the directory. -When using packages that are part of the Kubernetes source code, the -commands above are not enough because the [lack of semantic -versioning](https://github.com/kubernetes/kubernetes/issues/72638) -prevents `go mod` from finding newer releases. Importing directly from -`kubernetes/kubernetes` also needs `replace` statements to override -the fake `v0.0.0` versions -(https://github.com/kubernetes/kubernetes/issues/79384). The -`go-get-kubernetes.sh` script can be used to update all packages in -lockstep to a different Kubernetes version. It takes a single version -number like "1.16.0". - Conversion of a repository that uses `dep` to `go mod` can be done with: GO111MODULE=on go mod init @@ -160,3 +149,18 @@ Conversion of a repository that uses `dep` to `go mod` can be done with: GO111MODULE=on go mod vendor git rm -f Gopkg.toml Gopkg.lock git add go.mod go.sum vendor + +### Updating Kubernetes dependencies + +When using packages that are part of the Kubernetes source code, the +commands above are not enough because the [lack of semantic +versioning](https://github.com/kubernetes/kubernetes/issues/72638) +prevents `go mod` from finding newer releases. Importing directly from +`kubernetes/kubernetes` also needs `replace` statements to override +the fake `v0.0.0` versions +(https://github.com/kubernetes/kubernetes/issues/79384). The +`go-get-kubernetes.sh` script can be used to update all packages in +lockstep to a different Kubernetes version. Example usage: +``` +$ ./release-tools/go-get-kubernetes.sh 1.16.4 +``` From 4ad69492c97834b52d74d0b67b6f47b0590e4be1 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 3 Dec 2019 23:48:29 -0800 Subject: [PATCH 06/18] Improve snapshot pod running checks and improve version_gt Signed-off-by: Grant Griffiths --- prow.sh | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/prow.sh b/prow.sh index d719e9808d..27c7051294 100755 --- a/prow.sh +++ b/prow.sh @@ -713,10 +713,11 @@ install_snapshot_controller() { kubectl apply -f "https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/${CSI_SNAPSHOTTER_VERSION}/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml" cnt=0 - until kubectl get statefulset snapshot-controller | grep snapshot-controller | grep "1/1"; do + expected_running_pods=$(curl https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/"${CSI_SNAPSHOTTER_VERSION}"/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml | grep replicas | cut -d ':' -f 2-) + while [ "$(kubectl get pods -l app=snapshot-controller | grep 'Running' -c)" -lt "$expected_running_pods" ]; do if [ $cnt -gt 30 ]; then - echo "Running statefulsets:" - kubectl describe statefulsets + echo "snapshot-controller pod status:" + kubectl describe pods -l app=snapshot-controller echo >&2 "ERROR: snapshot controller not ready after over 5 min" exit 1 fi @@ -996,8 +997,30 @@ make_test_to_junit () { fi } +# version_gt returns true if arg1 is greater than arg2. +# +# This function expects versions to be one of the following formats: +# X.Y.Z, release-X.Y.Z, vX.Y.Z +# +# where X,Y, and Z are any number. +# +# Partial versions (1.2, release-1.2) work as well. +# The follow substrings are stripped before version comparison: +# - "v" +# - "release-" +# +# Usage: +# version_gt release-1.3 v1.2.0 (returns true) +# version_gt v1.1.1 v1.2.0 (returns false) +# version_gt 1.1.1 v1.2.0 (returns false) +# version_gt 1.3.1 v1.2.0 (returns true) +# version_gt 1.1.1 release-1.2.0 (returns false) +# version_gt 1.2.0 1.2.2 (returns false) function version_gt() { - test "$(printf '%s\n' "$@" | sort -V | head -n 1)" != "$1"; + versions=$(for ver in "$@"; do ver=${ver#release-}; echo "${ver#v}"; done) + greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"v"}; + test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" } main () { From a4e629966848d0097461f49f8aab0dfeb68d7a68 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 4 Dec 2019 14:12:50 -0800 Subject: [PATCH 07/18] fix syntax for ppc64le build --- build.make | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.make b/build.make index 7075a37eed..a9b9d25dc7 100644 --- a/build.make +++ b/build.make @@ -70,7 +70,7 @@ build-%: check-go-version-go CGO_ENABLED=0 GOOS=linux go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$* ./cmd/$* if [ "$$ARCH" = "amd64" ]; then \ CGO_ENABLED=0 GOOS=windows go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*.exe ./cmd/$* ; \ - CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* + CGO_ENABLED=0 GOOS=linux GOARCH=ppc64le go build $(GOFLAGS_VENDOR) -a -ldflags '-X main.version=$(REV) -extldflags "-static"' -o ./bin/$*-ppc64le ./cmd/$* ; \ fi container-%: build-% From b98b2aed087f4e768db673a9d43dff4c7abd1482 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Mon, 16 Dec 2019 19:13:38 -0800 Subject: [PATCH 08/18] Enable snapshot tests in 1.17 to be run in non-alpha jobs. This requires adding one more parallel e2e test run with a special focus flag because snapshot tests are still guarded with a "[Feature:VolumeSnapshotDataSource]" tag. The setting that skips all tests with "[Feature:.*]" has to be removed because it overrides the focus. We don't have serial snapshot tests yet. This needs to be modified again if we add any in the future. --- prow.sh | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/prow.sh b/prow.sh index 9bfdc2e288..c160f886c5 100755 --- a/prow.sh +++ b/prow.sh @@ -132,7 +132,7 @@ configvar CSI_PROW_BUILD_JOB true "building code in repo enabled" # use the same settings as for "latest" Kubernetes. This works # as long as there are no breaking changes in Kubernetes, like # deprecating or changing the implementation of an alpha feature. -configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" +configvar CSI_PROW_KUBERNETES_VERSION 1.17.0 "Kubernetes" # This is a hack to workaround the issue that each version # of kind currently only supports specific patch versions of @@ -142,7 +142,6 @@ configvar CSI_PROW_KUBERNETES_VERSION 1.15.3 "Kubernetes" # # If the version is prefixed with "release-", then nothing # is overridden. -override_k8s_version "1.14.6" override_k8s_version "1.15.3" # CSI_PROW_KUBERNETES_VERSION reduced to first two version numbers and @@ -206,9 +205,9 @@ configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" # # CSI_PROW_E2E_REPO=none disables E2E testing. # TOOO: remove versioned variables and make e2e version match k8s version -configvar CSI_PROW_E2E_VERSION_1_14 v1.14.0 "E2E version for Kubernetes 1.14.x" configvar CSI_PROW_E2E_VERSION_1_15 v1.15.0 "E2E version for Kubernetes 1.15.x" configvar CSI_PROW_E2E_VERSION_1_16 v1.16.0 "E2E version for Kubernetes 1.16.x" +configvar CSI_PROW_E2E_VERSION_1_17 v1.17.0 "E2E version for Kubernetes 1.17.x" # TODO: add new CSI_PROW_E2E_VERSION entry for future Kubernetes releases configvar CSI_PROW_E2E_VERSION_LATEST master "E2E version for Kubernetes master" # testing against Kubernetes master is already tracking a moving target, so we might as well use a moving E2E version configvar CSI_PROW_E2E_REPO_LATEST https://github.com/kubernetes/kubernetes "E2E repo for Kubernetes >= 1.13.x" # currently the same for all versions @@ -278,6 +277,14 @@ tests_need_alpha_cluster () { tests_enabled "parallel-alpha" "serial-alpha" } +# Regex for non-alpha, feature-tagged tests that should be run. +# +# Starting with 1.17, snapshots is beta, but the E2E tests still have the +# [Feature:] tag. They need to be explicitly enabled. +configvar CSI_PROW_E2E_FOCUS_1_15 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.15" # no tests to run, match nothing +configvar CSI_PROW_E2E_FOCUS_1_16 '^' "non-alpha, feature-tagged tests for Kubernetes = 1.16" # no tests to run, match nothing +configvar CSI_PROW_E2E_FOCUS_LATEST '\[Feature:VolumeSnapshotDataSource\]' "non-alpha, feature-tagged tests for Kubernetes >= 1.17" +configvar CSI_PROW_E2E_FOCUS "$(get_versioned_variable CSI_PROW_E2E_FOCUS "${csi_prow_kubernetes_version_suffix}")" "non-alpha, feature-tagged tests" # Serial vs. parallel is always determined by these regular expressions. # Individual regular expressions are seperated by spaces for readability @@ -313,12 +320,11 @@ configvar CSI_PROW_E2E_ALPHA "$(get_versioned_variable CSI_PROW_E2E_ALPHA "${csi # kubernetes-csi components must be updated, either by disabling # the failing test for "latest" or by updating the test and not running # it anymore for older releases. -configvar CSI_PROW_E2E_ALPHA_GATES_1_14 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.14" configvar CSI_PROW_E2E_ALPHA_GATES_1_15 'VolumeSnapshotDataSource=true,ExpandCSIVolumes=true' "alpha feature gates for Kubernetes 1.15" configvar CSI_PROW_E2E_ALPHA_GATES_1_16 'VolumeSnapshotDataSource=true' "alpha feature gates for Kubernetes 1.16" # TODO: add new CSI_PROW_ALPHA_GATES_xxx entry for future Kubernetes releases and # add new gates to CSI_PROW_E2E_ALPHA_GATES_LATEST. -configvar CSI_PROW_E2E_ALPHA_GATES_LATEST 'VolumeSnapshotDataSource=true' "alpha feature gates for latest Kubernetes" +configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kubernetes" configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment @@ -1111,6 +1117,16 @@ main () { warn "E2E parallel failed" ret=1 fi + + # Run tests that are feature tagged, but non-alpha + # Ignore: Double quote to prevent globbing and word splitting. + # shellcheck disable=SC2086 + if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ + -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then + warn "E2E parallel features failed" + ret=1 + fi fi if tests_enabled "serial"; then From fc80975954a9720a1611fa9a5195cbfc48b64167 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Fri, 20 Dec 2019 16:30:25 -0800 Subject: [PATCH 09/18] Fix version_gt to work with kubernetes prefix Signed-off-by: Grant Griffiths --- prow.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index c160f886c5..832da1ee50 100755 --- a/prow.sh +++ b/prow.sh @@ -1013,6 +1013,7 @@ make_test_to_junit () { # The follow substrings are stripped before version comparison: # - "v" # - "release-" +# - "kubernetes-" # # Usage: # version_gt release-1.3 v1.2.0 (returns true) @@ -1022,8 +1023,9 @@ make_test_to_junit () { # version_gt 1.1.1 release-1.2.0 (returns false) # version_gt 1.2.0 1.2.2 (returns false) function version_gt() { - versions=$(for ver in "$@"; do ver=${ver#release-}; echo "${ver#v}"; done) + versions=$(for ver in "$@"; do ver=${ver#release-}; ver=${ver#kubernetes-}; echo "${ver#v}"; done) greaterVersion=${1#"release-"}; + greaterVersion=${greaterVersion#"kubernetes-"}; greaterVersion=${greaterVersion#"v"}; test "$(printf '%s' "$versions" | sort -V | head -n 1)" != "$greaterVersion" } From af9549b5a11c4849ff7b4a1a42a7b8aea59137c3 Mon Sep 17 00:00:00 2001 From: saad-ali Date: Thu, 2 Jan 2020 14:29:40 -0800 Subject: [PATCH 10/18] Update prow hostpath driver version to 1.3.0-rc2 --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index 832da1ee50..ae51947ded 100755 --- a/prow.sh +++ b/prow.sh @@ -187,7 +187,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.2.0" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc2" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" From 8b0316c7e4d94a3206cdab0ea2e8d7f7b3be637b Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 2 Jan 2020 14:33:46 -0800 Subject: [PATCH 11/18] Fix overriding of junit results by using unique names for each e2e run --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index ae51947ded..e21a7ba9e3 100755 --- a/prow.sh +++ b/prow.sh @@ -1123,7 +1123,7 @@ main () { # Run tests that are feature tagged, but non-alpha # Ignore: Double quote to prevent globbing and word splitting. # shellcheck disable=SC2086 - if ! run_e2e parallel ${CSI_PROW_GINKO_PARALLEL} \ + if ! run_e2e parallel-features ${CSI_PROW_GINKO_PARALLEL} \ -focus="External.Storage.*($(regex_join "${CSI_PROW_E2E_FOCUS}"))" \ -skip="$(regex_join "${CSI_PROW_E2E_SERIAL}")"; then warn "E2E parallel features failed" From 8191eab6ffe694d11bf1e0d254330287c961dc77 Mon Sep 17 00:00:00 2001 From: Grant Griffiths Date: Tue, 7 Jan 2020 18:09:31 -0800 Subject: [PATCH 12/18] Update snapshotter to version v2.0.0 Signed-off-by: Grant Griffiths --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index e21a7ba9e3..eb451ef1b0 100755 --- a/prow.sh +++ b/prow.sh @@ -328,7 +328,7 @@ configvar CSI_PROW_E2E_ALPHA_GATES_LATEST '' "alpha feature gates for latest Kub configvar CSI_PROW_E2E_ALPHA_GATES "$(get_versioned_variable CSI_PROW_E2E_ALPHA_GATES "${csi_prow_kubernetes_version_suffix}")" "alpha E2E feature gates" # Which external-snapshotter tag to use for the snapshotter CRD and snapshot-controller deployment -configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0-rc4' "external-snapshotter version tag" +configvar CSI_SNAPSHOTTER_VERSION 'v2.0.0' "external-snapshotter version tag" # Some tests are known to be unusable in a KinD cluster. For example, # stopping kubelet with "ssh systemctl stop kubelet" simply From 6582f2ff3bd584e662035b7da2a90efc52184b1b Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Thu, 9 Jan 2020 17:25:38 -0800 Subject: [PATCH 13/18] Update hostpath driver version to get fix for connection-timeout --- prow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/prow.sh b/prow.sh index e21a7ba9e3..27309a7966 100755 --- a/prow.sh +++ b/prow.sh @@ -187,7 +187,7 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # # When no deploy script is found (nothing in `deploy` directory, # CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc2" "hostpath driver" +configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver" configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" From ac8a0212b93cad817689029be503839ad1959e21 Mon Sep 17 00:00:00 2001 From: Michelle Au Date: Wed, 11 Dec 2019 17:56:08 -0800 Subject: [PATCH 14/18] Document the process for releasing a new sidecar --- SIDECAR_RELEASE_PROCESS.md | 90 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 90 insertions(+) create mode 100644 SIDECAR_RELEASE_PROCESS.md diff --git a/SIDECAR_RELEASE_PROCESS.md b/SIDECAR_RELEASE_PROCESS.md new file mode 100644 index 0000000000..91a1e1b827 --- /dev/null +++ b/SIDECAR_RELEASE_PROCESS.md @@ -0,0 +1,90 @@ +# Sidecar Release Process + +This page describes the process for releasing a kubernetes-csi sidecar. + +## Prerequisites + +The release manager must: + +* Be a member of the kubernetes-csi organization. Open an + [issue](https://github.com/kubernetes/org/issues/new?assignees=&labels=area%2Fgithub-membership&template=membership.md&title=REQUEST%3A+New+membership+for+%3Cyour-GH-handle%3E) in + kubernetes/org to request membership +* Be a top level approver for the repository. To become a top level approver, + the candidate must demonstrate ownership and deep knowledge of the repository + through active maintainence, responding to and fixing issues, reviewing PRs, + test triage. +* Be part of the maintainers or admin group for the repository. admin is a + superset of maintainers, only maintainers level is required for cutting a + release. Membership can be requested by submitting a PR to kubernetes/org. + [Example](https://github.com/kubernetes/org/pull/1467) + +## Updating CI Jobs +Whenever a new Kubernetes minor version is released, our kubernetes-csi CI jobs +must be updated. + +[Our CI jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) have the +naming convention `-on-`. + +1. Jobs should be actively monitored to find and fix failures in sidecars and + infrastructure changes early in the development cycle. Test failures are sent + to kubernetes-sig-storage-test-failures@googlegroups.com. +1. "-on-master" jobs are the closest reflection to the new Kubernetes version. +1. Fixes to our prow.sh CI script can be tested in the [CSI hostpath + repo](https://github.com/kubernetes-csi/csi-driver-host-path) by modifying + [prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/release-tools/prow.sh) + along with any overrides in + [.prow.sh](https://github.com/kubernetes-csi/csi-driver-host-path/blob/master/.prow.sh) + to mirror the failing environment. Once e2e tests are passing (verify-unit tests + will fail), then the prow.sh changes can be submitted to [csi-release-tools](https://github.com/kubernetes-csi/csi-release-tools). +1. Changes can then be updated in all the sidecar repos and hostpath driver repo + by following the [update + instructions](https://github.com/kubernetes-csi/csi-release-tools/blob/master/README.md#sharing-and-updating). +1. New pull and CI jobs are configured by + [gen-jobs.sh](https://github.com/kubernetes/test-infra/blob/master/config/jobs/kubernetes-csi/gen-jobs.sh). + New pull jobs that have been unverified should be initially made optional. + [Example](https://github.com/kubernetes/test-infra/pull/15055) +1. Once new pull and CI jobs have been verified, and the new Kubernetes version + is released, we can make the optional jobs required, and also remove the + Kubernetes versions that are no longer supported. + +## Release Process +1. Identify all issues and ongoing PRs that should go into the release, and + drive them to resolution. +1. Download [K8s release notes + generator](https://github.com/kubernetes/release/tree/master/cmd/release-notes) +1. Generate release notes for the release. Replace arguments with the relevant + information. + ``` + GITHUB_TOKEN= ./release-notes --start-sha=0ed6978fd199e3ca10326b82b4b8b8e916211c9b --end-sha=3cb3d2f18ed8cb40371c6d8886edcabd1f27e7b9 \ + --github-org=kubernetes-csi --github-repo=external-attacher -branch=master -output out.md + ``` + * `--start-sha` should point to the last release from the same branch. For + example: + * `1.X-1.0` tag when releasing `1.X.0` + * `1.X.Y-1` tag when releasing `1.X.Y` +1. Compare the generated output to the new commits for the release to check if + any notable change missed a release note. +1. Reword release notes as needed. Make sure to check notes for breaking + changes and deprecations. +1. If release is a new major/minor version, create a new `CHANGELOG-..md` + file. Otherwise, add the release notes to the top of the existing CHANGELOG + file for that minor version. +1. Submit a PR for the CHANGELOG changes. +1. Submit a PR for README changes, in particular, Compatibility, Feature status, + and any other sections that may need updating. +1. Check that all [canary CI + jobs](https://k8s-testgrid.appspot.com/sig-storage-csi-ci) are passing, + and that test coverage is adequate for the changes that are going into the release. +1. Make sure that no new PRs have merged in the meantime, and no PRs are in + flight and soon to be merged. +1. Create a new release following a previous release as a template. Be sure to select the correct + branch. This requires Github release permissions as required by the prerequisites. + [external-provisioner example](https://github.com/kubernetes-csi/external-provisioner/releases/new) +1. If release was a new major/minor version, create a new `release-` + branch at that commit. +1. Update [kubernetes-csi/docs](https://github.com/kubernetes-csi/docs) sidecar + and feature pages with the new released version. +1. After all the sidecars have been released, update + CSI hostpath driver with the new sidecars in the [CSI repo](https://github.com/kubernetes-csi/csi-driver-host-path/tree/master/deploy) + and [k/k + in-tree](https://github.com/kubernetes/kubernetes/tree/master/test/e2e/testing-manifests/storage-csi/hostpath/hostpath) From fa90abd07862e6fa0a7b66ea2435b9db364ce6c9 Mon Sep 17 00:00:00 2001 From: wangzheng03 Date: Sun, 19 Jan 2020 09:42:26 +0800 Subject: [PATCH 15/18] fix incorrect link --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 08c828846c..60eab2a98e 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ on what is enabled in Prow, see https://github.com/kubernetes/test-infra/tree/master/config/jobs/kubernetes-csi Test results for periodic jobs are visible in -https://testgrid.k8s.io/sig-storage-csi +https://testgrid.k8s.io/sig-storage-csi-ci It is possible to reproduce the Prow testing locally on a suitable machine: - Linux host From ba077bc5c19a264a49a4d0f6ad30f2afd0028bf4 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 10 Feb 2020 11:06:31 +0100 Subject: [PATCH 16/18] prow.sh: also configure feature gates for kubelet That this hasn't been done before is an oversight. Apparently it hasn't been a problem because there never have been feature gates that mattered? --- release-tools/prow.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 118c5bd195..6778e8b8e0 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -568,6 +568,13 @@ kubeadmConfigPatches: nodeRegistration: kubeletExtraArgs: "feature-gates": "$gates" +- | + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + metadata: + name: config + featureGates: +$(list_gates "$gates") - | apiVersion: kubeproxy.config.k8s.io/v1alpha1 kind: KubeProxyConfiguration From 342ab6ea5138be57495a1861000cf11fc12fe035 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 10 Feb 2020 13:08:33 +0100 Subject: [PATCH 17/18] prow.sh: generic driver installation This relies on a slightly different deployment script: a "deploy.sh" must exist which knows that it has to dump a test driver configurion into the file pointed to with CSI_PROW_TEST_DRIVER, if that env variable is set. That way, we no longer need to know what capabilities the installed driver has. --- release-tools/prow.sh | 104 +++++++++++++++++------------------------- 1 file changed, 43 insertions(+), 61 deletions(-) diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 6778e8b8e0..0331ac3f88 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -157,7 +157,9 @@ csi_prow_kubernetes_version_suffix="$(echo "${CSI_PROW_KUBERNETES_VERSION}" | tr # the caller. configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csiprow.XXXXXXXXXX")" "work directory" -# The hostpath deployment script is searched for in several places. +# By default, this script tests sidecars with the CSI hostpath driver, +# using the install_csi_driver function. That function depends on +# a deployment script that it searches for in several places: # # - The "deploy" directory in the current repository: this is useful # for the situation that a component becomes incompatible with the @@ -165,11 +167,11 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # own example until the shared one can be updated; it's also how # csi-driver-host-path itself provides the example. # -# - CSI_PROW_HOSTPATH_VERSION of the CSI_PROW_HOSTPATH_REPO is checked +# - CSI_PROW_DRIVER_VERSION of the CSI_PROW_DRIVER_REPO is checked # out: this allows other repos to reference a version of the example # that is known to be compatible. # -# - The csi-driver-host-path/deploy directory has multiple sub-directories, +# - The /deploy directory can have multiple sub-directories, # each with different deployments (stable set of images for Kubernetes 1.13, # stable set of images for Kubernetes 1.14, canary for latest Kubernetes, etc.). # This is necessary because there may be incompatible changes in the @@ -186,16 +188,26 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # "none" disables the deployment of the hostpath driver. # # When no deploy script is found (nothing in `deploy` directory, -# CSI_PROW_HOSTPATH_REPO=none), nothing gets deployed. -configvar CSI_PROW_HOSTPATH_VERSION "v1.3.0-rc3" "hostpath driver" -configvar CSI_PROW_HOSTPATH_REPO https://github.com/kubernetes-csi/csi-driver-host-path "hostpath repo" +# CSI_PROW_DRIVER_REPO=none), nothing gets deployed. +# +# If the deployment script is called with CSI_PROW_TEST_DRIVER= as +# environment variable, then it must write a suitable test driver configuration +# into that file in addition to installing the driver. +configvar CSI_PROW_DRIVER_VERSION "v1.3.0-rc4" "CSI driver version" +configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" -configvar CSI_PROW_HOSTPATH_DRIVER_NAME "hostpath.csi.k8s.io" "the hostpath driver name" -# If CSI_PROW_HOSTPATH_CANARY is set (typically to "canary", but also -# "1.0-canary"), then all image versions are replaced with that -# version tag. -configvar CSI_PROW_HOSTPATH_CANARY "" "hostpath image" +# The install_csi_driver function may work also for other CSI drivers, +# as long as they follow the conventions of the CSI hostpath driver. +# If they don't, then a different install function can be provided in +# a .prow.sh file and this config variable can be overridden. +configvar CSI_PROW_DRIVER_INSTALL "install_csi_driver" "name of the shell function which installs the CSI driver" + +# If CSI_PROW_DRIVER_CANARY is set (typically to "canary", but also +# version tag. Usually empty. CSI_PROW_HOSTPATH_CANARY is +# accepted as alternative name because some test-infra jobs +# still use that name. +configvar CSI_PROW_DRIVER_CANARY "${CSI_PROW_HOSTPATH_CANARY}" "driver image override for canary images" # The E2E testing can come from an arbitrary repo. The expectation is that # the repo supports "go test ./test/e2e -args --storage.testdriver" (https://github.com/kubernetes/kubernetes/pull/72836) @@ -620,7 +632,7 @@ find_deployment () { # Fixed deployment name? Use it if it exists, otherwise fail. if [ "${CSI_PROW_DEPLOYMENT}" ]; then - file="$dir/${CSI_PROW_DEPLOYMENT}/deploy-hostpath.sh" + file="$dir/${CSI_PROW_DEPLOYMENT}/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -630,9 +642,9 @@ find_deployment () { # Ignore: See if you can use ${variable//search/replace} instead. # shellcheck disable=SC2001 - file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy-hostpath.sh" + file="$dir/kubernetes-$(echo "${CSI_PROW_KUBERNETES_VERSION}" | sed -e 's/\([0-9]*\)\.\([0-9]*\).*/\1.\2/')/deploy.sh" if ! [ -e "$file" ]; then - file="$dir/kubernetes-latest/deploy-hostpath.sh" + file="$dir/kubernetes-latest/deploy.sh" if ! [ -e "$file" ]; then return 1 fi @@ -640,12 +652,11 @@ find_deployment () { echo "$file" } -# This installs the hostpath driver example. CSI_PROW_HOSTPATH_CANARY overrides all -# image versions with that canary version. The parameters of install_hostpath can be -# used to override registry and/or tag of individual images (CSI_PROVISIONER_REGISTRY=localhost:9000 -# CSI_PROVISIONER_TAG=latest). -install_hostpath () { - local images deploy_hostpath +# This installs the CSI driver. It's called with a list of env variables +# that override the default images. CSI_PROW_DRIVER_CANARY overrides all +# image versions with that canary version. +install_csi_driver () { + local images deploy_driver images="$*" if [ "${CSI_PROW_DEPLOYMENT}" = "none" ]; then @@ -661,31 +672,31 @@ install_hostpath () { done fi - if deploy_hostpath="$(find_deployment "$(pwd)/deploy")"; then + if deploy_driver="$(find_deployment "$(pwd)/deploy")"; then : - elif [ "${CSI_PROW_HOSTPATH_REPO}" = "none" ]; then + elif [ "${CSI_PROW_DRIVER_REPO}" = "none" ]; then return 1 else - git_checkout "${CSI_PROW_HOSTPATH_REPO}" "${CSI_PROW_WORK}/hostpath" "${CSI_PROW_HOSTPATH_VERSION}" --depth=1 || die "checking out hostpath repo failed" - if deploy_hostpath="$(find_deployment "${CSI_PROW_WORK}/hostpath/deploy")"; then + git_checkout "${CSI_PROW_DRIVER_REPO}" "${CSI_PROW_WORK}/csi-driver" "${CSI_PROW_DRIVER_VERSION}" --depth=1 || die "checking out CSI driver repo failed" + if deploy_driver="$(find_deployment "${CSI_PROW_WORK}/csi-driver/deploy")"; then : else - die "deploy-hostpath.sh not found in ${CSI_PROW_HOSTPATH_REPO} ${CSI_PROW_HOSTPATH_VERSION}. To disable E2E testing, set CSI_PROW_HOSTPATH_REPO=none" + die "deploy.sh not found in ${CSI_PROW_DRIVER_REPO} ${CSI_PROW_DRIVER_VERSION}. To disable E2E testing, set CSI_PROW_DRIVER_REPO=none" fi fi - if [ "${CSI_PROW_HOSTPATH_CANARY}" != "stable" ]; then - images="$images IMAGE_TAG=${CSI_PROW_HOSTPATH_CANARY}" + if [ "${CSI_PROW_DRIVER_CANARY}" != "stable" ]; then + images="$images IMAGE_TAG=${CSI_PROW_DRIVER_CANARY}" fi # Ignore: Double quote to prevent globbing and word splitting. # It's intentional here for $images. # shellcheck disable=SC2086 - if ! run env $images "${deploy_hostpath}"; then + if ! run env "CSI_PROW_TEST_DRIVER=${CSI_PROW_WORK}/test-driver.yaml" $images "${deploy_driver}"; then # Collect information about failed deployment before failing. collect_cluster_info (start_loggers >/dev/null; wait) info "For container output see job artifacts." - die "deploying the hostpath driver with ${deploy_hostpath} failed" + die "deploying the CSI driver with ${deploy_driver} failed" fi } @@ -811,33 +822,6 @@ install_sanity () ( run_with_go "${CSI_PROW_GO_VERSION_SANITY}" go test -c -o "${CSI_PROW_WORK}/csi-sanity" "${CSI_PROW_SANITY_IMPORT_PATH}/cmd/csi-sanity" || die "building csi-sanity failed" ) -# The default implementation of this function generates a external -# driver test configuration for the hostpath driver. -# -# The content depends on both what the E2E suite expects and what the -# installed hostpath driver supports. Generating it here seems prone -# to breakage, but it is uncertain where a better place might be. -generate_test_driver () { - cat <"${CSI_PROW_WORK}/test-driver.yaml" || die "generating test-driver.yaml failed" - # Rename, merge and filter JUnit files. Necessary in case that we run the E2E suite again # and to avoid the large number of "skipped" tests that we get from using # the full Kubernetes E2E testsuite while only running a few tests. @@ -1070,7 +1052,7 @@ main () { cmds="$(grep '^\s*CMDS\s*=' Makefile | sed -e 's/\s*CMDS\s*=//')" # Get the image that was just built (if any) from the # top-level Makefile CMDS variable and set the - # deploy-hostpath.sh env variables for it. We also need to + # deploy.sh env variables for it. We also need to # side-load those images into the cluster. for i in $cmds; do e=$(echo "$i" | tr '[:lower:]' '[:upper:]' | tr - _) @@ -1108,7 +1090,7 @@ main () { fi # Installing the driver might be disabled. - if install_hostpath "$images"; then + if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info if sanity_enabled; then @@ -1165,7 +1147,7 @@ main () { fi # Installing the driver might be disabled. - if install_hostpath "$images"; then + if ${CSI_PROW_DRIVER_INSTALL} "$images"; then collect_cluster_info if tests_enabled "parallel-alpha"; then From e504c65dadd2e0ad1827559a43c4900bdc700cd2 Mon Sep 17 00:00:00 2001 From: Patrick Ohly Date: Mon, 10 Feb 2020 13:29:50 +0100 Subject: [PATCH 18/18] WIP: override hostpath driver repo for testing --- release-tools/prow.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/release-tools/prow.sh b/release-tools/prow.sh index 0331ac3f88..58711a65f4 100755 --- a/release-tools/prow.sh +++ b/release-tools/prow.sh @@ -193,8 +193,8 @@ configvar CSI_PROW_WORK "$(mkdir -p "$GOPATH/pkg" && mktemp -d "$GOPATH/pkg/csip # If the deployment script is called with CSI_PROW_TEST_DRIVER= as # environment variable, then it must write a suitable test driver configuration # into that file in addition to installing the driver. -configvar CSI_PROW_DRIVER_VERSION "v1.3.0-rc4" "CSI driver version" -configvar CSI_PROW_DRIVER_REPO https://github.com/kubernetes-csi/csi-driver-host-path "CSI driver repo" +configvar CSI_PROW_DRIVER_VERSION "test-driver-config" "CSI driver version" +configvar CSI_PROW_DRIVER_REPO https://github.com/pohly/csi-driver-host-path "CSI driver repo" configvar CSI_PROW_DEPLOYMENT "" "deployment" # The install_csi_driver function may work also for other CSI drivers,