diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 55ea00ea1660..f733ea8421ce 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,9 +1,11 @@ - -Thank you for contributing to Rook! --> - -**Description of your changes:** + **Which issue is resolved by this Pull Request:** Resolves # diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 18e4d5c1e6de..7734b3c62549 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -18,6 +18,23 @@ updates: patterns: - "github.com*" + # Dependencies listed in go.mod + - package-ecosystem: "gomod" + directory: "/pkg/apis" # Location of package manifests + schedule: + interval: "weekly" + groups: + golang-dependencies: + patterns: + - "github.com/golang*" + k8s-dependencies: + patterns: + - "k8s.io*" + - "sigs.k8s.io*" + github-dependencies: + patterns: + - "github.com*" + # Dependencies listed in .github/workflows/*.yml - package-ecosystem: "github-actions" directory: "/" diff --git a/.github/workflows/canary-integration-test.yml b/.github/workflows/canary-integration-test.yml index ba50167b32dd..8a751fddbd9b 100644 --- a/.github/workflows/canary-integration-test.yml +++ b/.github/workflows/canary-integration-test.yml @@ -283,7 +283,7 @@ jobs: - name: use local disk as OSD run: | tests/scripts/github-action-helper.sh use_local_disk - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - name: prepare loop devices for osds @@ -349,7 +349,7 @@ jobs: - name: use local disk as OSD run: | tests/scripts/github-action-helper.sh use_local_disk - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - name: deploy cluster @@ -394,7 +394,7 @@ jobs: - name: use local disk as OSD run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh use_local_disk tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only @@ -446,7 +446,7 @@ jobs: - name: use local disk as OSD run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh use_local_disk tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only @@ -492,13 +492,13 @@ jobs: - name: use local disk as OSD run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh use_local_disk tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --wipe-only - name: create LV on disk run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK - name: deploy cluster @@ -549,7 +549,7 @@ jobs: - name: create cluster prerequisites run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/localPathPV.sh "$BLOCK" tests/scripts/loopDevicePV.sh 1 tests/scripts/github-action-helper.sh create_cluster_prerequisites @@ -580,7 +580,7 @@ jobs: kubectl -n rook-ceph logs deploy/rook-ceph-operator tests/scripts/github-action-helper.sh wait_for_cleanup_pod lsblk - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) sudo head --bytes=60 ${BLOCK}1 sudo head --bytes=60 ${BLOCK}2 sudo head --bytes=60 /dev/loop1 @@ -716,7 +716,7 @@ jobs: - name: create cluster prerequisites run: | - tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy cluster @@ -742,7 +742,7 @@ jobs: kubectl -n rook-ceph delete cephcluster rook-ceph kubectl -n rook-ceph logs deploy/rook-ceph-operator tests/scripts/github-action-helper.sh wait_for_cleanup_pod - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) sudo head --bytes=60 ${BLOCK}1 sudo head --bytes=60 ${BLOCK}2 sudo lsblk @@ -889,7 +889,7 @@ jobs: - name: create cluster prerequisites run: | - tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy vault @@ -970,7 +970,7 @@ jobs: - name: create cluster prerequisites run: | - tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy vault @@ -1032,7 +1032,7 @@ jobs: - name: create LV on disk run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh create_LV_on_disk $BLOCK tests/scripts/localPathPV.sh /dev/test-rook-vg/test-rook-lv @@ -1080,7 +1080,7 @@ jobs: - name: use local disk into two partitions run: | tests/scripts/github-action-helper.sh use_local_disk - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 sudo lsblk diff --git a/.github/workflows/commitlint.yml b/.github/workflows/commitlint.yml index 216d12a25fd7..f624f052c8ff 100644 --- a/.github/workflows/commitlint.yml +++ b/.github/workflows/commitlint.yml @@ -31,7 +31,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - uses: wagoid/commitlint-github-action@v5.4.3 + - uses: wagoid/commitlint-github-action@v5.4.4 with: configFile: "./.commitlintrc.json" helpURL: https://rook.io/docs/rook/latest/Contributing/development-flow/#commit-structure diff --git a/.github/workflows/daily-nightly-jobs.yml b/.github/workflows/daily-nightly-jobs.yml index b06c94ed2772..fbd54e23c0a9 100644 --- a/.github/workflows/daily-nightly-jobs.yml +++ b/.github/workflows/daily-nightly-jobs.yml @@ -108,41 +108,6 @@ jobs: name: canary-arm64 path: test - smoke-suite-pacific-devel: - if: github.repository == 'rook/rook' - runs-on: ubuntu-20.04 - steps: - - name: checkout - uses: actions/checkout@v4 - with: - fetch-depth: 0 - - - name: setup cluster resources - uses: ./.github/workflows/integration-test-config-latest-k8s - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - kubernetes-version: "1.28.0" - - - name: TestCephSmokeSuite - run: | - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) - SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="pacific-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration - - - name: collect common logs - if: always() - run: | - export LOG_DIR="/home/runner/work/rook/rook/tests/integration/_output/tests/" - export CLUSTER_NAMESPACE="smoke-ns" - export OPERATOR_NAMESPACE="smoke-ns-system" - tests/scripts/collect-logs.sh - - - name: Artifact - uses: actions/upload-artifact@v3 - if: failure() - with: - name: ceph-smoke-suite-pacific-artifact - path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - smoke-suite-quincy-devel: if: github.repository == 'rook/rook' runs-on: ubuntu-20.04 @@ -160,7 +125,7 @@ jobs: - name: TestCephSmokeSuite run: | - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="quincy-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration - name: collect common logs @@ -195,7 +160,7 @@ jobs: - name: TestCephSmokeSuite run: | - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="reef-devel" go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration - name: collect common logs @@ -213,7 +178,6 @@ jobs: name: ceph-smoke-suite-reef-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - smoke-suite-ceph-main: if: github.repository == 'rook/rook' runs-on: ubuntu-20.04 @@ -231,7 +195,7 @@ jobs: - name: TestCephSmokeSuite run: | - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=main go test -v -timeout 1800s -run TestCephSmokeSuite github.com/rook/rook/tests/integration - name: collect common logs @@ -249,7 +213,7 @@ jobs: name: ceph-smoke-suite-master-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - object-suite-pacific-devel: + object-suite-quincy-devel: if: github.repository == 'rook/rook' runs-on: ubuntu-20.04 steps: @@ -266,8 +230,8 @@ jobs: - name: TestCephObjectSuite run: | - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) - SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="pacific-devel" go test -v -timeout 1800s -failfast -run TestCephObjectSuite github.com/rook/rook/tests/integration + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) + SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION="quincy-devel" go test -v -timeout 1800s -failfast -run TestCephObjectSuite github.com/rook/rook/tests/integration - name: collect common logs if: always() @@ -281,7 +245,7 @@ jobs: uses: actions/upload-artifact@v3 if: failure() with: - name: ceph-object-suite-pacific-artifact + name: ceph-object-suite-quincy-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ object-suite-ceph-main: @@ -301,7 +265,7 @@ jobs: - name: TestCephObjectSuite run: | - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_CLEANUP_POLICY=false CEPH_SUITE_VERSION=main go test -v -timeout 1800s -failfast -run TestCephObjectSuite github.com/rook/rook/tests/integration - name: collect common logs @@ -319,7 +283,7 @@ jobs: name: ceph-object-suite-master-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ - upgrade-from-pacific-stable-to-pacific-devel: + upgrade-from-reef-stable-to-reef-devel: if: github.repository == 'rook/rook' runs-on: ubuntu-20.04 steps: @@ -336,8 +300,8 @@ jobs: - name: TestCephUpgradeSuite run: | - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) - go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToPacificDevel github.com/rook/rook/tests/integration + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) + go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToReefDevel github.com/rook/rook/tests/integration - name: collect common logs if: always() @@ -351,7 +315,7 @@ jobs: uses: actions/upload-artifact@v3 if: failure() with: - name: ceph-upgrade-suite-pacific-artifact + name: ceph-upgrade-suite-reef-artifact path: /home/runner/work/rook/rook/tests/integration/_output/tests/ upgrade-from-quincy-stable-to-quincy-devel: @@ -371,7 +335,7 @@ jobs: - name: TestCephUpgradeSuite run: | - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) go test -v -timeout 1800s -failfast -run TestCephUpgradeSuite/TestUpgradeCephToQuincyDevel github.com/rook/rook/tests/integration - name: collect common logs @@ -395,7 +359,7 @@ jobs: strategy: matrix: ceph-image-tag: - ["latest-main-devel", "latest-quincy-devel", "latest-pacific-devel"] + ["latest-main-devel", "latest-quincy-devel", "latest-reef-devel"] steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml b/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml index 6d397a4a1f75..85e47c5b3430 100644 --- a/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml +++ b/.github/workflows/encryption-pvc-kms-ibm-kp/action.yml @@ -36,7 +36,7 @@ runs: - name: create cluster prerequisites shell: bash --noprofile --norc -eo pipefail -x {0} run: | - tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ {print $1}'| head -1) + tests/scripts/localPathPV.sh $(lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh create_cluster_prerequisites - name: deploy cluster diff --git a/.github/workflows/golangci-lint.yaml b/.github/workflows/golangci-lint.yaml index 1e9777e38228..0bae7e8e99da 100644 --- a/.github/workflows/golangci-lint.yaml +++ b/.github/workflows/golangci-lint.yaml @@ -49,5 +49,8 @@ jobs: name: govulncheck runs-on: ubuntu-latest steps: + - uses: actions/setup-go@v4 + with: + go-version: "1.21.4" - name: govulncheck uses: golang/govulncheck-action@v1 diff --git a/.github/workflows/helm-lint.yaml b/.github/workflows/helm-lint.yaml index 97ef9ad80fa3..33a176176373 100644 --- a/.github/workflows/helm-lint.yaml +++ b/.github/workflows/helm-lint.yaml @@ -40,7 +40,7 @@ jobs: python-version: 3.9 - name: Set up chart-testing - uses: helm/chart-testing-action@v2.6.0 + uses: helm/chart-testing-action@v2.6.1 - name: Run chart-testing (lint) run: ct lint --charts=./deploy/charts/rook-ceph --validate-yaml=false --validate-maintainers=false diff --git a/.github/workflows/integration-test-helm-suite.yaml b/.github/workflows/integration-test-helm-suite.yaml index ad9aa9fc81df..17fe29753856 100644 --- a/.github/workflows/integration-test-helm-suite.yaml +++ b/.github/workflows/integration-test-helm-suite.yaml @@ -51,7 +51,7 @@ jobs: tests/scripts/github-action-helper.sh collect_udev_logs_in_background tests/scripts/github-action-helper.sh create_helm_tag tests/scripts/helm.sh up - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_TEST_CLEANUP=false SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -failfast -run CephHelmSuite github.com/rook/rook/tests/integration - name: collect common logs diff --git a/.github/workflows/integration-test-mgr-suite.yaml b/.github/workflows/integration-test-mgr-suite.yaml index a3dfd0dc5759..e5e0c1cd32a3 100644 --- a/.github/workflows/integration-test-mgr-suite.yaml +++ b/.github/workflows/integration-test-mgr-suite.yaml @@ -46,7 +46,7 @@ jobs: - name: TestCephMgrSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) go test -v -timeout 1800s -failfast -run CephMgrSuite github.com/rook/rook/tests/integration - name: collect common logs diff --git a/.github/workflows/integration-test-multi-cluster-suite.yaml b/.github/workflows/integration-test-multi-cluster-suite.yaml index 9f5e376adf89..14f406427f67 100644 --- a/.github/workflows/integration-test-multi-cluster-suite.yaml +++ b/.github/workflows/integration-test-multi-cluster-suite.yaml @@ -46,8 +46,8 @@ jobs: - name: TestCephMultiClusterDeploySuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export TEST_SCRATCH_DEVICE=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1 - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export TEST_SCRATCH_DEVICE=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1)1 + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) go test -v -timeout 1800s -failfast -run CephMultiClusterDeploySuite github.com/rook/rook/tests/integration - name: collect common logs diff --git a/.github/workflows/integration-test-object-suite.yaml b/.github/workflows/integration-test-object-suite.yaml index 4970ef72c0db..a729b8154c0c 100644 --- a/.github/workflows/integration-test-object-suite.yaml +++ b/.github/workflows/integration-test-object-suite.yaml @@ -46,7 +46,7 @@ jobs: - name: TestCephObjectSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_CLEANUP_POLICY=false go test -v -timeout 2400s -failfast -run CephObjectSuite github.com/rook/rook/tests/integration - name: collect common logs diff --git a/.github/workflows/integration-test-smoke-suite.yaml b/.github/workflows/integration-test-smoke-suite.yaml index e8121cf5d74b..eef5d8e088e3 100644 --- a/.github/workflows/integration-test-smoke-suite.yaml +++ b/.github/workflows/integration-test-smoke-suite.yaml @@ -46,7 +46,7 @@ jobs: - name: TestCephSmokeSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -failfast -run CephSmokeSuite github.com/rook/rook/tests/integration - name: collect common logs diff --git a/.github/workflows/integration-test-upgrade-suite.yaml b/.github/workflows/integration-test-upgrade-suite.yaml index ed71a294560e..6ea0110ba0fa 100644 --- a/.github/workflows/integration-test-upgrade-suite.yaml +++ b/.github/workflows/integration-test-upgrade-suite.yaml @@ -46,7 +46,7 @@ jobs: - name: TestCephUpgradeSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) go test -v -timeout 2400s -failfast -run CephUpgradeSuite/TestUpgradeRook github.com/rook/rook/tests/integration - name: collect common logs @@ -94,7 +94,7 @@ jobs: tests/scripts/helm.sh up tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) go test -v -timeout 1800s -failfast -run CephUpgradeSuite/TestUpgradeHelm github.com/rook/rook/tests/integration - name: collect common logs diff --git a/.github/workflows/integration-tests-on-release.yaml b/.github/workflows/integration-tests-on-release.yaml index f9dacc9ce116..b1be3ab30ebf 100644 --- a/.github/workflows/integration-tests-on-release.yaml +++ b/.github/workflows/integration-tests-on-release.yaml @@ -36,7 +36,7 @@ jobs: tests/scripts/github-action-helper.sh collect_udev_logs_in_background tests/scripts/github-action-helper.sh create_helm_tag tests/scripts/helm.sh up - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_TEST_CLEANUP=false SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephHelmSuite github.com/rook/rook/tests/integration - name: collect common logs @@ -75,8 +75,8 @@ jobs: - name: TestCephMultiClusterDeploySuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export TEST_SCRATCH_DEVICE=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1)1 - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export TEST_SCRATCH_DEVICE=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1)1 + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) go test -v -timeout 1800s -run CephMultiClusterDeploySuite github.com/rook/rook/tests/integration - name: collect common logs @@ -115,7 +115,7 @@ jobs: - name: TestCephSmokeSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_CLEANUP_POLICY=false go test -v -timeout 1800s -run CephSmokeSuite github.com/rook/rook/tests/integration - name: collect common logs @@ -154,7 +154,7 @@ jobs: - name: TestCephUpgradeSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) go test -v -timeout 2400s -run CephUpgradeSuite/TestUpgradeRook github.com/rook/rook/tests/integration - name: collect common logs @@ -196,7 +196,7 @@ jobs: tests/scripts/helm.sh up tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) go test -v -timeout 1800s -run CephUpgradeSuite/TestUpgradeHelm github.com/rook/rook/tests/integration - name: collect common logs @@ -235,7 +235,7 @@ jobs: - name: TestCephObjectSuite run: | tests/scripts/github-action-helper.sh collect_udev_logs_in_background - export DEVICE_FILTER=$(lsblk|awk '/14G/ {print $1}'| head -1) + export DEVICE_FILTER=$(lsblk|awk '/14G/ || /64G/ {print $1}'| head -1) SKIP_CLEANUP_POLICY=false go test -v -timeout 2400s -failfast -run CephObjectSuite github.com/rook/rook/tests/integration - name: collect common logs diff --git a/.github/workflows/rgw-multisite-test/action.yml b/.github/workflows/rgw-multisite-test/action.yml index 2de4db5f7740..033ac3ab449c 100644 --- a/.github/workflows/rgw-multisite-test/action.yml +++ b/.github/workflows/rgw-multisite-test/action.yml @@ -24,7 +24,7 @@ runs: - name: use local disk into two partitions shell: bash --noprofile --norc -eo pipefail -x {0} run: | - BLOCK=$(sudo lsblk --paths|awk '/14G/ {print $1}'| head -1) + BLOCK=$(sudo lsblk --paths|awk '/14G/ || /64G/ {print $1}'| head -1) tests/scripts/github-action-helper.sh use_local_disk tests/scripts/create-bluestore-partitions.sh --disk "$BLOCK" --osd-count 2 sudo lsblk diff --git a/.github/workflows/unit-test.yml b/.github/workflows/unit-test.yml index 4fa86bcfc0c3..f42dc1067830 100644 --- a/.github/workflows/unit-test.yml +++ b/.github/workflows/unit-test.yml @@ -46,7 +46,7 @@ jobs: go-version: "1.21" - name: Setup jq - uses: dcarbone/install-jq-action@v2.0.2 + uses: dcarbone/install-jq-action@v2.1.0 with: version: "${{ inputs.version }}" diff --git a/Documentation/CRDs/Cluster/ceph-cluster-crd.md b/Documentation/CRDs/Cluster/ceph-cluster-crd.md index db26fa9e02a4..82935cd93c31 100755 --- a/Documentation/CRDs/Cluster/ceph-cluster-crd.md +++ b/Documentation/CRDs/Cluster/ceph-cluster-crd.md @@ -26,12 +26,12 @@ Settings can be specified at the global level to apply to the cluster as a whole * `external`: * `enable`: if `true`, the cluster will not be managed by Rook but via an external entity. This mode is intended to connect to an existing cluster. In this case, Rook will only consume the external cluster. However, Rook will be able to deploy various daemons in Kubernetes such as object gateways, mds and nfs if an image is provided and will refuse otherwise. If this setting is enabled **all** the other options will be ignored except `cephVersion.image` and `dataDirHostPath`. See [external cluster configuration](external-cluster.md). If `cephVersion.image` is left blank, Rook will refuse the creation of extra CRs like object, file and nfs. * `cephVersion`: The version information for launching the ceph daemons. - * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v16.2.11` or `v17.2.6`. For more details read the [container images section](#ceph-container-images). + * `image`: The image used for running the ceph daemons. For example, `quay.io/ceph/ceph:v17.2.6`. For more details read the [container images section](#ceph-container-images). For the latest ceph images, see the [Ceph DockerHub](https://hub.docker.com/r/ceph/ceph/tags/). To ensure a consistent version of the image is running across all nodes in the cluster, it is recommended to use a very specific image version. Tags also exist that would give the latest version, but they are only recommended for test environments. For example, the tag `v17` will be updated each time a new Quincy build is released. Using the `v17` tag is not recommended in production because it may lead to inconsistent versions of the image running across different nodes in the cluster. - * `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently `pacific`, `quincy`, and `reef` are supported. Future versions such as `squid` (v19) would require this to be set to `true`. Should be set to `false` in production. + * `allowUnsupported`: If `true`, allow an unsupported major version of the Ceph release. Currently `quincy` and `reef` are supported. Future versions such as `squid` (v19) would require this to be set to `true`. Should be set to `false` in production. `imagePullPolicy`: The image pull policy for the ceph daemon pods. Possible values are `Always`, `IfNotPresent`, and `Never`. The default is `IfNotPresent`. * `dataDirHostPath`: The path on the host ([hostPath](https://kubernetes.io/docs/concepts/storage/volumes/#hostpath)) where config and data should be stored for each of the services. If the directory does not exist, it will be created. Because this directory persists on the host, it will remain after pods are deleted. Following paths and any of their subpaths **must not be used**: `/etc/ceph`, `/rook` or `/var/log/ceph`. @@ -368,8 +368,8 @@ repository. At the time of writing it's unclear when this will be supported. #### IPFamily -Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. Kubernetes version should be at least v1.13 to run IPv6. Dual-stack is supported as of ceph Pacific. -To turn on dual stack see the [network configuration section](#network-configuration-settings). +Provide single-stack IPv4 or IPv6 protocol to assign corresponding addresses to pods and services. This field is optional. Possible inputs are IPv6 and IPv4. Empty value will be treated as IPv4. +To enable dual stack see the [network configuration section](#network-configuration-settings). ### Node Settings diff --git a/Documentation/CRDs/Cluster/stretch-cluster.md b/Documentation/CRDs/Cluster/stretch-cluster.md index b6374a662c0a..248f220e7ab2 100644 --- a/Documentation/CRDs/Cluster/stretch-cluster.md +++ b/Documentation/CRDs/Cluster/stretch-cluster.md @@ -34,7 +34,6 @@ spec: - name: b - name: c cephVersion: - # Stretch cluster is supported in Ceph Pacific or newer. image: quay.io/ceph/ceph:v17.2.6 allowUnsupported: true # Either storageClassDeviceSets or the storage section can be specified for creating OSDs. diff --git a/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md b/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md index 7824746e4bf2..4dde0d8bf1ac 100644 --- a/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md +++ b/Documentation/CRDs/Object-Storage/ceph-object-store-user-crd.md @@ -46,7 +46,7 @@ spec: * `capabilities`: Ceph allows users to be given additional permissions. Due to missing APIs in go-ceph for updating the user capabilities, this setting can currently only be used during the creation of the object store user. If a user's capabilities need modified, the user must be deleted and re-created. See the [Ceph docs](https://docs.ceph.com/en/latest/radosgw/admin/#add-remove-admin-capabilities) for more info. Rook supports adding `read`, `write`, `read, write`, or `*` permissions for the following resources: - * `users` + * `user` * `buckets` * `usage` * `metadata` diff --git a/Documentation/CRDs/specification.md b/Documentation/CRDs/specification.md index 37bc3bae59af..56137812a0cb 100644 --- a/Documentation/CRDs/specification.md +++ b/Documentation/CRDs/specification.md @@ -6135,8 +6135,7 @@ string (Optional)

The Ceph pool used store the shared configuration for NFS-Ganesha daemons. -This setting is required for Ceph v15 and ignored for Ceph v16. -As of Ceph Pacific 16.2.7+, this is internally hardcoded to “.nfs”.

+This setting is deprecated, as it is internally required to be “.nfs”.

@@ -6149,8 +6148,7 @@ string (Optional)

The namespace inside the Ceph pool (set by ‘pool’) where shared NFS-Ganesha config is stored. -This setting is required for Ceph v15 and ignored for Ceph v16. -As of Ceph Pacific v16+, this is internally set to the name of the CephNFS.

+This setting is deprecated as it is internally set to the name of the CephNFS.

@@ -6274,6 +6272,21 @@ bool

Whether host networking is enabled for the Ganesha server. If not set, the network settings from the cluster CR will be applied.

+ + +livenessProbe
+ + +ProbeSpec + + + + +(Optional) +

A liveness-probe to verify that Ganesha server has valid run-time state. +If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe.

+ +

GatewaySpec @@ -10014,7 +10027,7 @@ QuotaSpec

ProbeSpec

-(Appears on:MetadataServerSpec, ObjectHealthCheckSpec) +(Appears on:GaneshaServerSpec, MetadataServerSpec, ObjectHealthCheckSpec)

ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon

diff --git a/Documentation/Contributing/development-environment.md b/Documentation/Contributing/development-environment.md index 2c8b4ad176a0..2a15fca123aa 100644 --- a/Documentation/Contributing/development-environment.md +++ b/Documentation/Contributing/development-environment.md @@ -95,4 +95,6 @@ To accelerate the development process, users have the option to employ the scrip at `tests/scripts/create-dev-cluster.sh`. This script is designed to rapidly set up a new minikube environment, apply the CRDs and the common file, and then utilize the `cluster-test.yaml` script to create the Rook cluster. Once setup, users can use the different `*-test.yaml` -files from the `deploy/examples/` directory to configure their clusters. +files from the `deploy/examples/` directory to configure their clusters. This script supports +the possibility of creating multiple rook clusters running on the same machine by using the option +`-p `. diff --git a/Documentation/Helm-Charts/operator-chart.md b/Documentation/Helm-Charts/operator-chart.md index e7b5f06eb58b..a8731f4919e5 100644 --- a/Documentation/Helm-Charts/operator-chart.md +++ b/Documentation/Helm-Charts/operator-chart.md @@ -61,7 +61,6 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.cephFSPluginUpdateStrategy` | CSI CephFS plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` | | `csi.cephFSPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI cephFS plugin daemonset update strategy. | `1` | | `csi.cephcsi.image` | Ceph CSI image | `quay.io/cephcsi/cephcsi:v3.9.0` | -| `csi.cephfsGrpcMetricsPort` | CSI CephFS driver GRPC metrics port | `9091` | | `csi.cephfsLivenessMetricsPort` | CSI CephFS driver metrics port | `9081` | | `csi.cephfsPodLabels` | Labels to add to the CSI CephFS Deployments and DaemonSets Pods | `nil` | | `csi.clusterName` | Cluster name identifier to set as metadata on the CephFS subvolume and RBD images. This will be useful in cases like for example, when two container orchestrator clusters (Kubernetes/OCP) are using a single ceph cluster | `nil` | @@ -82,7 +81,6 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.enableCSIHostNetwork` | Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary in some network configurations where the SDN does not provide access to an external cluster or there is significant drop in read/write performance | `true` | | `csi.enableCephfsDriver` | Enable Ceph CSI CephFS driver | `true` | | `csi.enableCephfsSnapshotter` | Enable Snapshotter in CephFS provisioner pod | `true` | -| `csi.enableGrpcMetrics` | Enable Ceph CSI GRPC Metrics | `false` | | `csi.enableLiveness` | Enable Ceph CSI Liveness sidecar deployment | `false` | | `csi.enableMetadata` | Enable adding volume metadata on the CephFS subvolumes and RBD images. Not all users might be interested in getting volume/snapshot details as metadata on CephFS subvolume and RBD images. Hence enable metadata is false by default | `false` | | `csi.enableNFSSnapshotter` | Enable Snapshotter in NFS provisioner pod | `true` | @@ -110,7 +108,6 @@ The following table lists the configurable parameters of the rook-operator chart | `csi.provisionerTolerations` | Array of tolerations in YAML format which will be added to CSI provisioner deployment | `nil` | | `csi.rbdAttachRequired` | Whether to skip any attach operation altogether for RBD PVCs. See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). If set to false it skips the volume attachments and makes the creation of pods using the RBD PVC fast. **WARNING** It's highly discouraged to use this for RWO volumes as it can cause data corruption. csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set to false since we'll have no VolumeAttachments to determine which node the PVC is mounted on. Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. | `true` | | `csi.rbdFSGroupPolicy` | Policy for modifying a volume's ownership or permissions when the RBD PVC is being mounted. supported values are documented at https://kubernetes-csi.github.io/docs/support-fsgroup.html | `"File"` | -| `csi.rbdGrpcMetricsPort` | Ceph CSI RBD driver GRPC metrics port | `9090` | | `csi.rbdLivenessMetricsPort` | Ceph CSI RBD driver metrics port | `8080` | | `csi.rbdPluginUpdateStrategy` | CSI RBD plugin daemonset update strategy, supported values are OnDelete and RollingUpdate | `RollingUpdate` | | `csi.rbdPluginUpdateStrategyMaxUnavailable` | A maxUnavailable parameter of CSI RBD plugin daemonset update strategy. | `1` | diff --git a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md index 4bab0c1b858e..43eed829fd4c 100644 --- a/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md +++ b/Documentation/Storage-Configuration/Ceph-CSI/ceph-csi-drivers.md @@ -48,7 +48,7 @@ csi_liveness 1 ``` Check the [monitoring doc](../Monitoring/ceph-monitoring.md) to see how to integrate CSI -liveness and grpc metrics into ceph monitoring. +liveness metrics into ceph monitoring. ## Dynamically Expand Volume @@ -253,6 +253,7 @@ Execute the following steps: * Patch the `rook-ceph-operator-config` configmap using the following command. + ```console kubectl patch cm rook-ceph-operator-config -nrook-ceph -p $'data:\n "CSI_ENABLE_READ_AFFINITY": "true"' ``` diff --git a/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md index b5f9b30ba778..7cb6e352ff07 100644 --- a/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md +++ b/Documentation/Storage-Configuration/Monitoring/ceph-monitoring.md @@ -261,7 +261,7 @@ After this you only need to create the service monitor as stated above. ### CSI Liveness -To integrate CSI liveness and grpc into ceph monitoring we will need to deploy +To integrate CSI liveness into ceph monitoring we will need to deploy a service and service monitor. ```console diff --git a/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md index 05a13a4861be..496275c99b28 100644 --- a/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md +++ b/Documentation/Storage-Configuration/Shared-Filesystem-CephFS/filesystem-storage.md @@ -10,10 +10,6 @@ This example runs a shared filesystem for the [kube-registry](https://github.com This guide assumes you have created a Rook cluster as explained in the main [quickstart guide](../../Getting-Started/quickstart.md) -### Multiple Filesystems Support - -Multiple filesystems are supported as of the Ceph Pacific release. - ## Create the Filesystem Create the filesystem by specifying the desired settings for the metadata pool, data pools, and metadata server in the `CephFilesystem` CRD. In this example we create the metadata pool with replication of three and a single data pool with replication of three. For more options, see the documentation on [creating shared filesystems](../../CRDs/Shared-Filesystem/ceph-filesystem-crd.md). diff --git a/Documentation/Upgrade/ceph-upgrade.md b/Documentation/Upgrade/ceph-upgrade.md index a302940b3081..a8441fdd43a0 100644 --- a/Documentation/Upgrade/ceph-upgrade.md +++ b/Documentation/Upgrade/ceph-upgrade.md @@ -24,14 +24,13 @@ until all the daemons have been updated. ## Supported Versions -Rook v1.12 supports the following Ceph versions: +Rook v1.13 supports the following Ceph versions: * Ceph Reef v18.2.0 or newer * Ceph Quincy v17.2.0 or newer -* Ceph Pacific v16.2.7 or newer -Support for Ceph Pacific (16.2.x) will be removed in the next Rook release. It will be mandatory to -upgrade to Quincy or Reef before upgrading to the Rook release after v1.12.x. +Support for Ceph Pacific (16.2.x) is removed in Rook v1.13. Upgrade to Quincy or Reef before upgrading +to Rook v1.13. !!! important When an update is requested, the operator will check Ceph's status, diff --git a/PendingReleaseNotes.md b/PendingReleaseNotes.md index 7d3ad56f100d..2de13db9ed27 100644 --- a/PendingReleaseNotes.md +++ b/PendingReleaseNotes.md @@ -2,4 +2,6 @@ ## Breaking Changes +- Removed support for Ceph Pacific (v16) + ## Features diff --git a/build/makelib/golang.mk b/build/makelib/golang.mk index fddadee82916..7bda5f098aec 100644 --- a/build/makelib/golang.mk +++ b/build/makelib/golang.mk @@ -170,6 +170,12 @@ go.mod.update: .PHONY: go.mod.check go.mod.check: + @echo === syncing root modules with APIs modules + @cp -a go.sum pkg/apis/go.sum + @cat go.mod | sed -e 's|^module github.com/rook/rook|module github.com/rook/rook/pkg/apis|' \ + -e '\:^replace github.com/rook/rook/pkg/apis => ./pkg/apis:d' > pkg/apis/go.mod + @echo === ensuring APIs modules are tidied + @(cd pkg/apis/; $(GOHOST) mod tidy -compat=$(GO_VERSION)) @echo === ensuring root modules are tidied @$(GOHOST) mod tidy -compat=$(GO_VERSION) diff --git a/build/release/Makefile b/build/release/Makefile index a771a5d860b6..6c9011d6319b 100644 --- a/build/release/Makefile +++ b/build/release/Makefile @@ -137,7 +137,7 @@ publish.docs: if test "$$GITHUB_ACTIONS" = "true"; then git config --unset 'http.https://github.com/.extraheader'; fi git fetch -u docs gh-pages # Switch to root of repo and then run the build and deploy of the documentation - cd $(ROOT_DIR) && mike deploy --remote docs --push --rebase --branch gh-pages --update-aliases --prefix $(DOCS_PREFIX) $(DOCS_VERSION) $(DOCS_VERSION_ALIAS) + cd $(ROOT_DIR) && mike deploy --remote docs --push --branch gh-pages --update-aliases --deploy-prefix $(DOCS_PREFIX) $(DOCS_VERSION) $(DOCS_VERSION_ALIAS) # ==================================================================================== # helm diff --git a/cmd/rook/userfacing/multus/validation/validation.go b/cmd/rook/userfacing/multus/validation/validation.go index d22005783186..57ef73159ee2 100644 --- a/cmd/rook/userfacing/multus/validation/validation.go +++ b/cmd/rook/userfacing/multus/validation/validation.go @@ -134,6 +134,18 @@ func init() { runCmd.Flags().StringVar(&validationConfig.NginxImage, "nginx-image", defaultConfig.NginxImage, "The Nginx image used for the validation server and clients.") + validationConfig.FlakyThreshold = defaultConfig.FlakyThreshold + f := (*timeoutSeconds)(&validationConfig.FlakyThreshold) + runCmd.Flags().VarPF(f, "flaky-threshold-seconds", "", + "This is the time window in which validation clients are all expected to become 'Ready' together. Validation clients are all started "+ + "at approximately the same time, and they should all stabilize at approximately the same time. Once the first validation client "+ + "becomes 'Ready', the tool checks that all of the remaining clients become 'Ready' before this threshold duration elapses. In networks "+ + "that have connectivity issues, limited bandwidth, or high latency, clients will contend for network traffic with each other, causing some "+ + "clients to randomly fail and become 'Ready' later than others. These randomly-failing clients are considered 'flaky.' Adjust this value "+ + "to reflect expectations for the underlying network. For fast and reliable networks, this can be set to a smaller value. For networks that "+ + "are intended to be slow, this can be set to a larger value. Additionally, for very large Kubernetes clusters, it may take longer for all "+ + "clients to start, and it therefore may take longer for all clients to become 'Ready'; in that case, this value can be set slightly higher.") + runCmd.Flags().StringVarP(&validationConfigFile, "config", "c", "", "The validation test config file to use. This cannot be used with other flags.") runCmd.MarkFlagsMutuallyExclusive("config", "timeout-minutes") @@ -142,6 +154,7 @@ func init() { runCmd.MarkFlagsMutuallyExclusive("config", "cluster-network") runCmd.MarkFlagsMutuallyExclusive("config", "daemons-per-node") runCmd.MarkFlagsMutuallyExclusive("config", "nginx-image") + runCmd.MarkFlagsMutuallyExclusive("config", "flaky-threshold-seconds") // flags for 'validation cleanup' // none @@ -253,3 +266,21 @@ func (t *timeoutMinutes) Set(v string) error { func (t timeoutMinutes) Type() string { return "timeoutMinutes" } + +type timeoutSeconds time.Duration + +func (t *timeoutSeconds) String() string { return time.Duration(*t).String() } +func (t *timeoutSeconds) Set(v string) error { + i, err := strconv.Atoi(v) + if err != nil { + return err + } + if i < 1 { + return fmt.Errorf("timeout must be greater than 0") + } + *t = timeoutSeconds(time.Duration(i) * time.Second) + return nil +} +func (t timeoutSeconds) Type() string { + return "timeoutSeconds" +} diff --git a/deploy/charts/rook-ceph-cluster/values.yaml b/deploy/charts/rook-ceph-cluster/values.yaml index d1c027301c33..67ed48f4bc41 100644 --- a/deploy/charts/rook-ceph-cluster/values.yaml +++ b/deploy/charts/rook-ceph-cluster/values.yaml @@ -88,13 +88,13 @@ cephClusterSpec: # For more details, check https://rook.io/docs/rook/v1.10/CRDs/Cluster/ceph-cluster-crd/ cephVersion: # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw). - # v16 is Pacific, v17 is Quincy. - # RECOMMENDATION: In production, use a specific version tag instead of the general v16 flag, which pulls the latest release and could result in different + # v17 is Quincy, v18 is Reef. + # RECOMMENDATION: In production, use a specific version tag instead of the general v18 flag, which pulls the latest release and could result in different # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/. - # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v15.2.11-20200419 + # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v17.2.6-20230410 # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities image: quay.io/ceph/ceph:v17.2.6 - # Whether to allow unsupported versions of Ceph. Currently `pacific`, `quincy`, and `reef` are supported. + # Whether to allow unsupported versions of Ceph. Currently `quincy`, and `reef` are supported. # Future versions such as `squid` (v19) would require this to be set to `true`. # Do not set to true in production. allowUnsupported: false diff --git a/deploy/charts/rook-ceph/templates/configmap.yaml b/deploy/charts/rook-ceph/templates/configmap.yaml index 440bcf8540c1..d23f3a022054 100644 --- a/deploy/charts/rook-ceph/templates/configmap.yaml +++ b/deploy/charts/rook-ceph/templates/configmap.yaml @@ -61,7 +61,6 @@ data: {{- if .Values.csi.kubeletDirPath }} ROOK_CSI_KUBELET_DIR_PATH: {{ .Values.csi.kubeletDirPath | quote }} {{- end }} - ROOK_CSI_ENABLE_GRPC_METRICS: {{ .Values.csi.enableGrpcMetrics | quote }} {{- if .Values.csi.cephcsi }} {{- if .Values.csi.cephcsi.image }} ROOK_CSI_CEPH_IMAGE: {{ .Values.csi.cephcsi.image | quote }} @@ -176,18 +175,12 @@ data: {{- if .Values.csi.nfsPluginNodeAffinity }} CSI_NFS_PLUGIN_NODE_AFFINITY: {{ .Values.csi.nfsPluginNodeAffinity }} {{- end }} -{{- if .Values.csi.cephfsGrpcMetricsPort }} - CSI_CEPHFS_GRPC_METRICS_PORT: {{ .Values.csi.cephfsGrpcMetricsPort | quote }} -{{- end }} {{- if .Values.csi.cephfsLivenessMetricsPort }} CSI_CEPHFS_LIVENESS_METRICS_PORT: {{ .Values.csi.cephfsLivenessMetricsPort | quote }} {{- end }} {{- if .Values.csi.enableLiveness }} CSI_ENABLE_LIVENESS: {{ .Values.csi.enableLiveness | quote }} {{- end }} -{{- if .Values.csi.rbdGrpcMetricsPort }} - CSI_RBD_GRPC_METRICS_PORT: {{ .Values.csi.rbdGrpcMetricsPort | quote }} -{{- end }} {{- if .Values.csi.rbdLivenessMetricsPort }} CSI_RBD_LIVENESS_METRICS_PORT: {{ .Values.csi.rbdLivenessMetricsPort | quote }} {{- end }} diff --git a/deploy/charts/rook-ceph/templates/resources.yaml b/deploy/charts/rook-ceph/templates/resources.yaml index a1c2b07fffa5..4a9ba1e33081 100644 --- a/deploy/charts/rook-ceph/templates/resources.yaml +++ b/deploy/charts/rook-ceph/templates/resources.yaml @@ -8002,10 +8002,10 @@ spec: nullable: true properties: namespace: - description: The namespace inside the Ceph pool (set by 'pool') where shared NFS-Ganesha config is stored. This setting is required for Ceph v15 and ignored for Ceph v16. As of Ceph Pacific v16+, this is internally set to the name of the CephNFS. + description: The namespace inside the Ceph pool (set by 'pool') where shared NFS-Ganesha config is stored. This setting is deprecated as it is internally set to the name of the CephNFS. type: string pool: - description: The Ceph pool used store the shared configuration for NFS-Ganesha daemons. This setting is required for Ceph v15 and ignored for Ceph v16. As of Ceph Pacific 16.2.7+, this is internally hardcoded to ".nfs". + description: The Ceph pool used store the shared configuration for NFS-Ganesha daemons. This setting is deprecated, as it is internally required to be ".nfs". type: string type: object security: @@ -9161,6 +9161,115 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + livenessProbe: + description: A liveness-probe to verify that Ganesha server has valid run-time state. If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object logLevel: description: LogLevel set logging level type: string diff --git a/deploy/charts/rook-ceph/templates/servicemonitor.yaml b/deploy/charts/rook-ceph/templates/servicemonitor.yaml index a9634d0e6548..b25dbe848850 100644 --- a/deploy/charts/rook-ceph/templates/servicemonitor.yaml +++ b/deploy/charts/rook-ceph/templates/servicemonitor.yaml @@ -21,9 +21,4 @@ spec: path: /metrics interval: {{ .Values.csi.serviceMonitor.interval }} {{- end }} - {{- if .Values.csi.enableGrpcMetrics }} - - port: csi-grpc-metrics - path: /metrics - interval: {{ .Values.csi.serviceMonitor.interval }} - {{- end }} {{- end }} diff --git a/deploy/charts/rook-ceph/values.yaml b/deploy/charts/rook-ceph/values.yaml index 638b42c3c300..e77e0ff4ae7e 100644 --- a/deploy/charts/rook-ceph/values.yaml +++ b/deploy/charts/rook-ceph/values.yaml @@ -82,8 +82,6 @@ csi: enableRbdDriver: true # -- Enable Ceph CSI CephFS driver enableCephfsDriver: true - # -- Enable Ceph CSI GRPC Metrics - enableGrpcMetrics: false # -- Enable host networking for CSI CephFS and RBD nodeplugins. This may be necessary # in some network configurations where the SDN does not provide access to an external cluster or # there is significant drop in read/write performance @@ -448,18 +446,10 @@ csi: # -- Enable Ceph CSI Liveness sidecar deployment enableLiveness: false - # -- CSI CephFS driver GRPC metrics port - # @default -- `9091` - cephfsGrpcMetricsPort: - # -- CSI CephFS driver metrics port # @default -- `9081` cephfsLivenessMetricsPort: - # -- Ceph CSI RBD driver GRPC metrics port - # @default -- `9090` - rbdGrpcMetricsPort: - # -- CSI Addons server port # @default -- `9070` csiAddonsPort: diff --git a/deploy/examples/cluster-stretched-aws.yaml b/deploy/examples/cluster-stretched-aws.yaml index f70164967bee..32260ffbe203 100644 --- a/deploy/examples/cluster-stretched-aws.yaml +++ b/deploy/examples/cluster-stretched-aws.yaml @@ -44,7 +44,6 @@ spec: mgr: count: 2 cephVersion: - # Stretch cluster support upstream is only available starting in Ceph Pacific image: quay.io/ceph/ceph:v17.2.6 allowUnsupported: true skipUpgradeChecks: false diff --git a/deploy/examples/cluster-stretched.yaml b/deploy/examples/cluster-stretched.yaml index f14c4c40d4d0..2e8e0c09a83d 100644 --- a/deploy/examples/cluster-stretched.yaml +++ b/deploy/examples/cluster-stretched.yaml @@ -38,7 +38,6 @@ spec: mgr: count: 2 cephVersion: - # Stretch cluster support upstream is only available starting in Ceph Pacific image: quay.io/ceph/ceph:v17.2.6 allowUnsupported: true skipUpgradeChecks: false diff --git a/deploy/examples/cluster.yaml b/deploy/examples/cluster.yaml index 28a99549f0f1..2a5f565f3179 100644 --- a/deploy/examples/cluster.yaml +++ b/deploy/examples/cluster.yaml @@ -16,13 +16,13 @@ metadata: spec: cephVersion: # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw). - # v16 is Pacific, and v17 is Quincy. + # v17 is Quincy, v18 is Reef. # RECOMMENDATION: In production, use a specific version tag instead of the general v17 flag, which pulls the latest release and could result in different # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/. - # If you want to be more precise, you can always use a timestamp tag such quay.io/ceph/ceph:v17.2.6-20230410 + # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v17.2.6-20230410 # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities image: quay.io/ceph/ceph:v17.2.6 - # Whether to allow unsupported versions of Ceph. Currently `pacific`, `quincy`, and `reef` are supported. + # Whether to allow unsupported versions of Ceph. Currently `quincy` and `reef` are supported. # Future versions such as `squid` (v19) would require this to be set to `true`. # Do not set to true in production. allowUnsupported: false diff --git a/deploy/examples/crds.yaml b/deploy/examples/crds.yaml index b557dc0ea0ed..5b3783a8490b 100644 --- a/deploy/examples/crds.yaml +++ b/deploy/examples/crds.yaml @@ -7995,10 +7995,10 @@ spec: nullable: true properties: namespace: - description: The namespace inside the Ceph pool (set by 'pool') where shared NFS-Ganesha config is stored. This setting is required for Ceph v15 and ignored for Ceph v16. As of Ceph Pacific v16+, this is internally set to the name of the CephNFS. + description: The namespace inside the Ceph pool (set by 'pool') where shared NFS-Ganesha config is stored. This setting is deprecated as it is internally set to the name of the CephNFS. type: string pool: - description: The Ceph pool used store the shared configuration for NFS-Ganesha daemons. This setting is required for Ceph v15 and ignored for Ceph v16. As of Ceph Pacific 16.2.7+, this is internally hardcoded to ".nfs". + description: The Ceph pool used store the shared configuration for NFS-Ganesha daemons. This setting is deprecated, as it is internally required to be ".nfs". type: string type: object security: @@ -9154,6 +9154,115 @@ spec: nullable: true type: object x-kubernetes-preserve-unknown-fields: true + livenessProbe: + description: A liveness-probe to verify that Ganesha server has valid run-time state. If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: Probe describes a health check to be performed against a container to determine whether it is alive or ready to receive traffic. + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: Command is the command line to execute inside the container, the working directory for the command is root ('/') in the container's filesystem. The command is simply exec'd, it is not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use a shell, you need to explicitly call out to that shell. Exit status of 0 is treated as live/healthy and non-zero is unhealthy. + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. Defaults to 3. Minimum value is 1. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: "Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md). \n If this is not specified, the default behavior is defined by gRPC." + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. You probably want to set "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name. This will be canonicalized upon output, so case-variant names will be understood as the same header. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: 'Number of seconds after the container has started before liveness probes are initiated. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. Name must be an IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. The grace period is the duration in seconds after the processes running in the pod are sent a termination signal and the time when the processes are forcibly halted with a kill signal. Set this value longer than the expected cleanup time for your process. If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this value overrides the value provided by the pod spec. Value must be non-negative integer. The value zero indicates stop immediately via the kill signal (no opportunity to shut down). This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset. + format: int64 + type: integer + timeoutSeconds: + description: 'Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes' + format: int32 + type: integer + type: object + type: object logLevel: description: LogLevel set logging level type: string diff --git a/deploy/examples/monitoring/csi-metrics-service-monitor.yaml b/deploy/examples/monitoring/csi-metrics-service-monitor.yaml index ea223d53ffd3..f1f2675f939a 100644 --- a/deploy/examples/monitoring/csi-metrics-service-monitor.yaml +++ b/deploy/examples/monitoring/csi-metrics-service-monitor.yaml @@ -17,7 +17,3 @@ spec: - port: csi-http-metrics path: /metrics interval: 5s - # comment csi-grpc-metrics related information if csi grpc metrics is not enabled - - port: csi-grpc-metrics - path: /metrics - interval: 5s diff --git a/deploy/examples/multus-validation.yaml b/deploy/examples/multus-validation.yaml index 8abaea964067..4e143d8d3f32 100644 --- a/deploy/examples/multus-validation.yaml +++ b/deploy/examples/multus-validation.yaml @@ -25,14 +25,15 @@ # rook multus validation run [--public-network=] [--cluster-network=] [flags] # # Flags: -# --cluster-network string The name of the Network Attachment Definition (NAD) that will be used for Ceph's cluster network. This should be a namespaced name in the form / if the NAD is defined in a different namespace from the cluster namespace. -# -c, --config string The validation test config file to use. This cannot be used with other flags. -# --daemons-per-node int The number of validation test daemons to run per node. It is recommended to set this to the maximum number of Ceph daemons that can run on any node in the worst case of node failure(s). The default value is set to the worst-case value for a Rook Ceph cluster with 3 portable OSDs, 3 portable monitors, and where all optional child resources have been created with 1 daemon such that they all might run on a single node in a failure scenario. If you aren't sure what to choose for this value, add 1 for each additional OSD beyond 3. (default 19) -# -h, --help help for run -# -n, --namespace string The namespace for validation test resources. It is recommended to set this to the namespace in which Rook's Ceph cluster will be installed. (default "rook-ceph") -# --nginx-image string The Nginx image used for the validation server and clients. (default "nginxinc/nginx-unprivileged:stable-alpine") -# --public-network string The name of the Network Attachment Definition (NAD) that will be used for Ceph's public network. This should be a namespaced name in the form / if the NAD is defined in a different namespace from the cluster namespace. -# --timeout-minutes timeoutMinutes The time to wait for resources to change to the expected state. For example, for the test web server to start, for test clients to become ready, or for test resources to be deleted. At longest, this may need to reflect the time it takes for client pods to to pull images, get address assignments, and then for each client to determine that its network connection is stable. Minimum: 1 minute. Recommended: 2 minutes or more. (default 3m0s) +# --cluster-network string The name of the Network Attachment Definition (NAD) that will be used for Ceph's cluster network. This should be a namespaced name in the form / if the NAD is defined in a different namespace from the cluster namespace. +# -c, --config string The validation test config file to use. This cannot be used with other flags. +# --daemons-per-node int The number of validation test daemons to run per node. It is recommended to set this to the maximum number of Ceph daemons that can run on any node in the worst case of node failure(s). The default value is set to the worst-case value for a Rook Ceph cluster with 3 portable OSDs, 3 portable monitors, and where all optional child resources have been created with 1 daemon such that they all might run on a single node in a failure scenario. If you aren't sure what to choose for this value, add 1 for each additional OSD beyond 3. (default 19) +# --flaky-threshold-seconds timeoutSeconds This is the time window in which validation clients are all expected to become 'Ready' together. Validation clients are all started at approximately the same time, and they should all stabilize at approximately the same time. Once the first validation client becomes 'Ready', the tool checks that all of the remaining clients become 'Ready' before this threshold duration elapses. In networks that have connectivity issues, limited bandwidth, or high latency, clients will contend for network traffic with each other, causing some clients to randomly fail and become 'Ready' later than others. These randomly-failing clients are considered 'flaky.' Adjust this value to reflect expectations for the underlying network. For fast and reliable networks, this can be set to a smaller value. For networks that are intended to be slow, this can be set to a larger value. Additionally, for very large Kubernetes clusters, it may take longer for all clients to start, and it therefore may take longer for all clients to become 'Ready'; in that case, this value can be set slightly higher. (default 30s) +# -h, --help help for run +# -n, --namespace string The namespace for validation test resources. It is recommended to set this to the namespace in which Rook's Ceph cluster will be installed. (default "rook-ceph") +# --nginx-image string The Nginx image used for the validation server and clients. (default "nginxinc/nginx-unprivileged:stable-alpine") +# --public-network string The name of the Network Attachment Definition (NAD) that will be used for Ceph's public network. This should be a namespaced name in the form / if the NAD is defined in a different namespace from the cluster namespace. +# --timeout-minutes timeoutMinutes The time to wait for resources to change to the expected state. For example, for the test web server to start, for test clients to become ready, or for test resources to be deleted. At longest, this may need to reflect the time it takes for client pods to to pull images, get address assignments, and then for each client to determine that its network connection is stable. Minimum: 1 minute. Recommended: 2 minutes or more. (default 3m0s) # # Global Flags: # --log-level string logging level for logging/tracing output (valid values: ERROR,WARNING,INFO,DEBUG) (default "INFO") diff --git a/deploy/examples/nfs.yaml b/deploy/examples/nfs.yaml index f5b755f43759..b43ae3033fd7 100644 --- a/deploy/examples/nfs.yaml +++ b/deploy/examples/nfs.yaml @@ -59,6 +59,10 @@ spec: # The logging levels: NIV_NULL | NIV_FATAL | NIV_MAJ | NIV_CRIT | NIV_WARN | NIV_EVENT | NIV_INFO | NIV_DEBUG | NIV_MID_DEBUG |NIV_FULL_DEBUG |NB_LOG_LEVEL logLevel: NIV_INFO + # Allow liveness-probe via pod's nfs port (TCP 2049) + # livenessProbe: + # disabled: false + # Configure security options for the NFS cluster. See docs for more information: # https://rook.github.io/docs/rook/latest/Storage-Configuration/NFS/nfs-security/ security: diff --git a/deploy/examples/operator-openshift.yaml b/deploy/examples/operator-openshift.yaml index 5c402ebe63de..125112d9a587 100644 --- a/deploy/examples/operator-openshift.yaml +++ b/deploy/examples/operator-openshift.yaml @@ -121,7 +121,6 @@ data: ROOK_CSI_ENABLE_RBD: "true" # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. ROOK_CSI_ENABLE_NFS: "false" - ROOK_CSI_ENABLE_GRPC_METRICS: "false" # Set to true to enable Ceph CSI pvc encryption support. CSI_ENABLE_ENCRYPTION: "false" @@ -549,13 +548,11 @@ data: # memory: 1Gi # cpu: 500m - # Configure CSI Ceph FS grpc and liveness metrics port + # Configure CSI CephFS liveness metrics port # Set to true to enable Ceph CSI liveness container. CSI_ENABLE_LIVENESS: "false" - # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" - # Configure CSI RBD grpc and liveness metrics port - # CSI_RBD_GRPC_METRICS_PORT: "9090" + # Configure CSI RBD liveness metrics port # CSI_RBD_LIVENESS_METRICS_PORT: "9080" # CSIADDONS_PORT: "9070" diff --git a/deploy/examples/operator.yaml b/deploy/examples/operator.yaml index 808f7cc777dd..2d980a5f43ba 100644 --- a/deploy/examples/operator.yaml +++ b/deploy/examples/operator.yaml @@ -35,7 +35,6 @@ data: ROOK_CSI_ENABLE_RBD: "true" # Enable the CSI NFS driver. To start another version of the CSI driver, see image properties below. ROOK_CSI_ENABLE_NFS: "false" - ROOK_CSI_ENABLE_GRPC_METRICS: "false" # Set to true to enable Ceph CSI pvc encryption support. CSI_ENABLE_ENCRYPTION: "false" @@ -476,13 +475,11 @@ data: # memory: 1Gi # cpu: 500m - # Configure CSI Ceph FS grpc and liveness metrics port + # Configure CSI CephFS liveness metrics port # Set to true to enable Ceph CSI liveness container. CSI_ENABLE_LIVENESS: "false" - # CSI_CEPHFS_GRPC_METRICS_PORT: "9091" # CSI_CEPHFS_LIVENESS_METRICS_PORT: "9081" - # Configure CSI RBD grpc and liveness metrics port - # CSI_RBD_GRPC_METRICS_PORT: "9090" + # Configure CSI RBD liveness metrics port # CSI_RBD_LIVENESS_METRICS_PORT: "9080" # CSIADDONS_PORT: "9070" diff --git a/go.mod b/go.mod index 36ea1922b0fb..ba24732bfb55 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,11 @@ module github.com/rook/rook go 1.20 +replace github.com/rook/rook/pkg/apis => ./pkg/apis + require ( github.com/IBM/keyprotect-go-client v0.12.2 - github.com/aws/aws-sdk-go v1.46.6 + github.com/aws/aws-sdk-go v1.47.9 github.com/banzaicloud/k8s-objectmatcher v1.8.0 github.com/ceph/go-ceph v0.24.0 github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8 @@ -18,16 +20,16 @@ require ( github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1 - github.com/openshift/api v0.0.0-20230816181854-a7ca92db022a github.com/pkg/errors v0.9.1 - github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 - github.com/prometheus-operator/prometheus-operator/pkg/client v0.68.0 - github.com/spf13/cobra v1.7.0 + github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.69.1 + github.com/prometheus-operator/prometheus-operator/pkg/client v0.69.1 + github.com/rook/rook/pkg/apis v0.0.0-00010101000000-000000000000 + github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/sykesm/zap-logfmt v0.0.4 go.uber.org/zap v1.26.0 - golang.org/x/sync v0.4.0 + golang.org/x/sync v0.5.0 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.28.3 @@ -55,7 +57,6 @@ require ( github.com/emicklei/go-restful/v3 v3.11.0 // indirect github.com/evanphx/json-patch v5.6.0+incompatible // indirect github.com/evanphx/json-patch/v5 v5.6.0 // indirect - github.com/fatih/color v1.15.0 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gemalto/flume v0.13.1 // indirect github.com/go-errors/errors v1.4.2 // indirect @@ -65,7 +66,6 @@ require ( github.com/go-openapi/jsonpointer v0.20.0 // indirect github.com/go-openapi/jsonreference v0.20.2 // indirect github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-test/deep v1.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -76,7 +76,6 @@ require ( github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/hashicorp/go-hclog v1.5.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.4 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect @@ -104,6 +103,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/openshift/api v0.0.0-20230816181854-a7ca92db022a // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_golang v1.16.0 // indirect diff --git a/go.sum b/go.sum index a62cf180bcc7..d80886716748 100644 --- a/go.sum +++ b/go.sum @@ -450,8 +450,8 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkY github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= -github.com/aws/aws-sdk-go v1.46.6 h1:6wFnNC9hETIZLMf6SOTN7IcclrOGwp/n9SLp8Pjt6E8= -github.com/aws/aws-sdk-go v1.46.6/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/aws/aws-sdk-go v1.47.9 h1:rarTsos0mA16q+huicGx0e560aYRtOucV5z2Mw23JRY= +github.com/aws/aws-sdk-go v1.47.9/go.mod h1:LF8svs817+Nz+DmiMQKTO3ubZ/6IaTpq3TjupRn3Eqk= github.com/banzaicloud/k8s-objectmatcher v1.8.0 h1:Nugn25elKtPMTA2br+JgHNeSQ04sc05MDPmpJnd1N2A= github.com/banzaicloud/k8s-objectmatcher v1.8.0/go.mod h1:p2LSNAjlECf07fbhDyebTkPUIYnU05G+WfGgkTmgeMg= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= @@ -510,7 +510,7 @@ github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8 h1:NrLmX9HDyGvQhyZdrDx8 github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/csi-addons/kubernetes-csi-addons v0.7.0 h1:Cqon7e9Wnil8gq4/M80uilMJcTv7LypdKJoOVhEb4oM= @@ -557,9 +557,7 @@ github.com/evanphx/json-patch/v5 v5.0.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2Vvl github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -655,9 +653,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= -github.com/go-test/deep v1.1.0 h1:WOcxcdHcvdgThNXjw0t76K42FXTU7HpNQWHpA2HHNlg= -github.com/go-test/deep v1.1.0/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= github.com/gobuffalo/flect v0.2.0/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= @@ -813,9 +810,8 @@ github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtng github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs= github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= -github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= -github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= @@ -921,8 +917,6 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -931,7 +925,6 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= @@ -1024,10 +1017,10 @@ github.com/portworx/dcos-secrets v0.0.0-20180616013705-8e8ec3f66611/go.mod h1:4h github.com/portworx/kvdb v0.0.0-20200929023115-b312c7519467/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0 h1:yl9ceUSUBo9woQIO+8eoWpcxZkdZgm89g+rVvu37TUw= -github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.68.0/go.mod h1:9Uuu3pEU2jB8PwuqkHvegQ0HV/BlZRJUyfTYAqfdVF8= -github.com/prometheus-operator/prometheus-operator/pkg/client v0.68.0 h1:8FS0sXpFkFPxp2gfkxyEMnhZV9yhf7xPbpsIeUZHlzM= -github.com/prometheus-operator/prometheus-operator/pkg/client v0.68.0/go.mod h1:ul4ND0BMCcOX1OSZvbJA1/lh7yQ2ILHNKuZIojGISe4= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.69.1 h1:hOnp+1FLBm+ifsyiRbunmfSs99jKAq+Tr5elCmo5l5U= +github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.69.1/go.mod h1:JtflYMUMay9HGil4aRg+dSj6X6mngtuBJf/ULOCxbxI= +github.com/prometheus-operator/prometheus-operator/pkg/client v0.69.1 h1:hCqId2xt6kOPuoeTxVxV/GiKmZyCJjYeuRRL90qcwH0= +github.com/prometheus-operator/prometheus-operator/pkg/client v0.69.1/go.mod h1:XfInnOa8WYXOlzOb0Iw+9eg0s/bkeOnXyten1pKFjuc= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -1082,8 +1075,8 @@ github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.0.0/go.mod h1:/6GTrnGXV9HjY+aR4k0oJ5tcvakLuG6EuKReYlHNrgE= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= @@ -1106,7 +1099,6 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= @@ -1355,8 +1347,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1430,7 +1422,6 @@ golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/images/ceph/Dockerfile b/images/ceph/Dockerfile index 2776be4b3ad2..26f6dbb052be 100644 --- a/images/ceph/Dockerfile +++ b/images/ceph/Dockerfile @@ -19,6 +19,9 @@ FROM BASEIMAGE ARG S5CMD_VERSION ARG S5CMD_ARCH +# install 'ip' tool for Multus +RUN dnf install -y --setopt=install_weak_deps=False iproute && dnf clean all + # Install the s5cmd package to interact with s3 gateway RUN curl --fail -sSL -o /s5cmd.tar.gz https://github.com/peak/s5cmd/releases/download/v${S5CMD_VERSION}/s5cmd_${S5CMD_VERSION}_${S5CMD_ARCH}.tar.gz && \ mkdir /s5cmd && \ diff --git a/pkg/apis/ceph.rook.io/v1/types.go b/pkg/apis/ceph.rook.io/v1/types.go index 5db6c1b1e5e5..baf337effa92 100755 --- a/pkg/apis/ceph.rook.io/v1/types.go +++ b/pkg/apis/ceph.rook.io/v1/types.go @@ -2065,14 +2065,12 @@ type NFSGaneshaSpec struct { // GaneshaRADOSSpec represents the specification of a Ganesha RADOS object type GaneshaRADOSSpec struct { // The Ceph pool used store the shared configuration for NFS-Ganesha daemons. - // This setting is required for Ceph v15 and ignored for Ceph v16. - // As of Ceph Pacific 16.2.7+, this is internally hardcoded to ".nfs". + // This setting is deprecated, as it is internally required to be ".nfs". // +optional Pool string `json:"pool,omitempty"` // The namespace inside the Ceph pool (set by 'pool') where shared NFS-Ganesha config is stored. - // This setting is required for Ceph v15 and ignored for Ceph v16. - // As of Ceph Pacific v16+, this is internally set to the name of the CephNFS. + // This setting is deprecated as it is internally set to the name of the CephNFS. // +optional Namespace string `json:"namespace,omitempty"` } @@ -2118,6 +2116,11 @@ type GaneshaServerSpec struct { // +nullable // +optional HostNetwork *bool `json:"hostNetwork,omitempty"` + + // A liveness-probe to verify that Ganesha server has valid run-time state. + // If LivenessProbe.Disabled is false and LivenessProbe.Probe is nil uses default probe. + // +optional + LivenessProbe *ProbeSpec `json:"livenessProbe,omitempty"` } // NFSSecuritySpec represents security configurations for an NFS server pod diff --git a/pkg/apis/go.mod b/pkg/apis/go.mod new file mode 100644 index 000000000000..09ef4e229688 --- /dev/null +++ b/pkg/apis/go.mod @@ -0,0 +1,125 @@ +module github.com/rook/rook/pkg/apis + +go 1.20 + +require ( + github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8 + github.com/hashicorp/vault/api v1.10.0 + github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 + github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 + github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1 + github.com/openshift/api v0.0.0-20230816181854-a7ca92db022a + github.com/pkg/errors v0.9.1 + github.com/stretchr/testify v1.8.4 + k8s.io/api v0.28.3 + k8s.io/apimachinery v0.28.3 + sigs.k8s.io/controller-runtime v0.16.3 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v3 v3.2.2 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/containernetworking/cni v1.1.2 // indirect + github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/evanphx/json-patch/v5 v5.6.0 // indirect + github.com/fatih/color v1.12.0 // indirect + github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/go-jose/go-jose/v3 v3.0.0 // indirect + github.com/go-logr/logr v1.2.4 // indirect + github.com/go-openapi/jsonpointer v0.20.0 // indirect + github.com/go-openapi/jsonreference v0.20.2 // indirect + github.com/go-openapi/swag v0.22.4 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/gofuzz v1.2.0 // indirect + github.com/google/uuid v1.4.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-retryablehttp v0.7.4 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 // indirect + github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect + github.com/hashicorp/go-sockaddr v1.0.2 // indirect + github.com/hashicorp/hcl v1.0.1-vault-5 // indirect + github.com/hashicorp/vault/api/auth/approle v0.5.0 // indirect + github.com/hashicorp/vault/api/auth/kubernetes v0.5.0 // indirect + github.com/imdario/mergo v0.3.16 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + github.com/ryanuber/go-glob v1.0.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect + go.uber.org/zap v1.26.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect + golang.org/x/text v0.13.0 // indirect + golang.org/x/time v0.3.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/inf.v0 v0.9.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/client-go v0.28.3 // indirect + k8s.io/klog/v2 v2.100.1 // indirect + k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f // indirect + k8s.io/utils v0.0.0-20230726121419-3b25d923346b // indirect + sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect + sigs.k8s.io/structured-merge-diff/v4 v4.3.0 // indirect + sigs.k8s.io/yaml v1.4.0 // indirect +) + +exclude ( + // exclude goproxy versions with security bugs + github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e + github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153 + github.com/elazarl/goproxy v0.0.0-20181111060418-2ce16c963a8a + // portworx dependencies are a mess, and we don't use portworx code, so skip it + github.com/portworx/sched-ops v1.20.4-rc1 + // Exclude pre-go-mod kubernetes tags, because they are older + // than v0.x releases but are picked when updating dependencies. + k8s.io/client-go v1.4.0 + k8s.io/client-go v1.5.0 + k8s.io/client-go v1.5.1 + k8s.io/client-go v1.5.2 + k8s.io/client-go v10.0.0+incompatible + k8s.io/client-go v11.0.0+incompatible + k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible + k8s.io/client-go v12.0.0+incompatible + k8s.io/client-go v2.0.0+incompatible + k8s.io/client-go v2.0.0-alpha.1+incompatible + k8s.io/client-go v3.0.0+incompatible + k8s.io/client-go v3.0.0-beta.0+incompatible + k8s.io/client-go v4.0.0+incompatible + k8s.io/client-go v4.0.0-beta.0+incompatible + k8s.io/client-go v5.0.0+incompatible + k8s.io/client-go v5.0.1+incompatible + k8s.io/client-go v6.0.0+incompatible + k8s.io/client-go v7.0.0+incompatible + k8s.io/client-go v8.0.0+incompatible + k8s.io/client-go v9.0.0+incompatible + k8s.io/client-go v9.0.0-invalid+incompatible +) diff --git a/pkg/apis/go.sum b/pkg/apis/go.sum new file mode 100644 index 000000000000..ff1b897f37d4 --- /dev/null +++ b/pkg/apis/go.sum @@ -0,0 +1,1102 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU= +cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY= +cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc= +cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0= +cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= +cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= +cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= +cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= +cloud.google.com/go v0.94.1/go.mod h1:qAlAugsXlC+JWO+Bke5vCtc9ONxjQT3drlTTnAplMW4= +cloud.google.com/go v0.97.0/go.mod h1:GF7l59pYBVlXQIBLx3a761cZ41F9bBH3JUlihCt2Udc= +cloud.google.com/go v0.99.0/go.mod h1:w0Xx2nLzqWJPuozYQX+hFfCSI8WioryfRDzkoI/Y2ZA= +cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A= +cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= +cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= +cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTBXtfbBFow= +cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM= +cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M= +cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz/FMzPu0s= +cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= +cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= +cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= +cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= +cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= +cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= +cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= +cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= +cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= +github.com/Azure/azure-sdk-for-go v62.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/Azure/go-autorest/autorest v0.11.18/go.mod h1:dSiJPy22c3u0OtOKDNttNgqpNFY/GeWa7GH/Pz56QRA= +github.com/Azure/go-autorest/autorest v0.11.27/go.mod h1:7l8ybrIdUmGqZMTD0sRtAr8NvbHjfofbf8RSP2q7w7U= +github.com/Azure/go-autorest/autorest/adal v0.9.13/go.mod h1:W/MM4U6nLxnIskrw4UwWzlHfGjwUS50aOsc/I3yuU8M= +github.com/Azure/go-autorest/autorest/adal v0.9.18/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/adal v0.9.20/go.mod h1:XVVeme+LZwABT8K5Lc3hA4nAe8LDBVle26gTrguhhPQ= +github.com/Azure/go-autorest/autorest/date v0.3.0/go.mod h1:BI0uouVdmngYNUzGWeSYnokU+TrmwEsOqdt8Y6sso74= +github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= +github.com/Azure/go-autorest/autorest/mocks v0.4.2/go.mod h1:Vy7OitM9Kei0i1Oj+LvyAWMXJHeKH1MVlzFugfVrmyU= +github.com/Azure/go-autorest/autorest/to v0.4.0/go.mod h1:fE8iZBn7LQR7zH/9XU2NcPR4o9jEImooCeWJcYV/zLE= +github.com/Azure/go-autorest/autorest/validation v0.3.1/go.mod h1:yhLgjC0Wda5DYXl6JAsWyUe4KVNffhoDhG0zVzUMo3E= +github.com/Azure/go-autorest/logger v0.2.1/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= +github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/IBM/keyprotect-go-client v0.5.1/go.mod h1:5TwDM/4FRJq1ZOlwQL1xFahLWQ3TveR88VmL1u3njyI= +github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= +github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.44.164/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cenkalti/backoff/v3 v3.0.0/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/cenkalti/backoff/v3 v3.2.2 h1:cfUAAO3yvKMYKPrvhDuHSwQnhZNk/RMHKdZqKTxfm6M= +github.com/cenkalti/backoff/v3 v3.2.2/go.mod h1:cIeZDE3IrqwwJl6VUwCN6trj1oXrTS4rc0ij+ULvLYs= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl31EQbXALQ= +github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf h1:iW4rZ826su+pqaw19uhpSCzhj44qo35pNgKFGqzDKkU= +github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8 h1:NrLmX9HDyGvQhyZdrDx89zCvPdxQ/EHCo+xGNrjNmHc= +github.com/coreos/pkg v0.0.0-20230601102743-20bbbf26f4d8/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= +github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs= +github.com/emicklei/go-restful/v3 v3.8.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.9.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= +github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= +github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= +github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/evanphx/json-patch v4.12.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch v5.6.0+incompatible h1:jBYDEEiFBPxA0v50tFdvOzQQTCvpL6mnFh5mB2/l16U= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.12.0 h1:mRhaKNwANqRgUBGKmnI5ZxEk7QXmjQeCcuYFMX2bfcc= +github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= +github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= +github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/getkin/kin-openapi v0.76.0/go.mod h1:660oXbgy5JFMKreazJaQTw7o+X00qeSyhcnluiMv+Xg= +github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo= +github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8= +github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= +github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/zapr v1.2.4 h1:QHVo+6stLbfJmYGkQ7uGHUCu5hnAFAj6mDe6Ea0SeOo= +github.com/go-openapi/jsonpointer v0.19.2/go.mod h1:3akKfEdA7DF1sugOqz1dVQHBcuDBPKZGEoHC/NkiQRg= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= +github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= +github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= +github.com/go-openapi/jsonreference v0.19.2/go.mod h1:jMjeRr2HHw6nAVajTXJ4eiUwohSTlpa0o73RUL1owJc= +github.com/go-openapi/jsonreference v0.19.3/go.mod h1:rjx6GuL8TTa9VaixXglHmQmIL98+wF9xc8zWvFonSJ8= +github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= +github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= +github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= +github.com/go-openapi/spec v0.19.3/go.mod h1:FpwSN1ksY1eteniUU7X0N/BgJ7a4WvBFVA8Lj9mJglo= +github.com/go-openapi/swag v0.19.2/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= +github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-test/deep v1.0.2/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/go-test/deep v1.0.8 h1:TDsG77qcSprGbC6vTN8OuXp5g+J+b5Pcguhf7Zt61VM= +github.com/go-test/deep v1.0.8/go.mod h1:5C2ZWiW0ErCdrYzpqxLbTX7MG14M9iiw8DgHncVwcsE= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/gnostic v0.5.7-v3refs/go.mod h1:73MKFl6jIHelAJNaBGFzt3SPtZULs9dYrGFt8OiIsHQ= +github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= +github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1 h1:K6RDEckDVWvDI9JAJYCmNdQXq6neHJOYx3V6jnqNEec= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= +github.com/googleapis/gax-go/v2 v2.1.1/go.mod h1:hddJymUZASv3XPyGkUpKj8pPO47Rmb0eJc8R6ouapiM= +github.com/googleapis/gax-go/v2 v2.2.0/go.mod h1:as02EH8zWkzwUoLbBaFeQ+arQaj/OthfcblKl4IGNaM= +github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99EXz9pXxye9YM= +github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= +github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= +github.com/googleapis/gnostic v0.5.1/go.mod h1:6U4PtQXGIEt/Z3h5MAT7FNofLnw9vXk2cUuW7uA/OeU= +github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v0.16.2 h1:K4ev2ib4LdQETX5cSZBG0DVLk1jwGqSPXBjdah3veNs= +github.com/hashicorp/go-hclog v0.16.2/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.6.2/go.mod h1:gEx6HMUGxYYhJScX7W1Il64m6cc2C1mDaW3NQ9sY1FY= +github.com/hashicorp/go-retryablehttp v0.6.6/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7 h1:UpiO20jno/eV1eVZcxqWnUohyKRe1g8FPV/xH1s/2qs= +github.com/hashicorp/go-secure-stdlib/parseutil v0.1.7/go.mod h1:QmrqtbKuxxSWTN3ETMPuB+VtEiBJ/A9XhoYGv8E1uD8= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.1/go.mod h1:gKOamz3EwoIoJq7mlMIRBpVTAUn8qPCrEclOKKWhD3U= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 h1:kes8mmyCpxJsI7FTwtzRqEy9CdjCtrXrXGuOpxEA7Ts= +github.com/hashicorp/go-secure-stdlib/strutil v0.1.2/go.mod h1:Gou2R9+il93BqX25LAKCLuM+y9U2T4hlwvT1yprcna4= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl v1.0.1-vault-5 h1:kI3hhbbyzr4dldA8UdTb7ZlVVlI2DACdCfz31RPDgJM= +github.com/hashicorp/hcl v1.0.1-vault-5/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= +github.com/hashicorp/vault/api v1.10.0 h1:/US7sIjWN6Imp4o/Rj1Ce2Nr5bki/AXi9vAW3p2tOJQ= +github.com/hashicorp/vault/api v1.10.0/go.mod h1:jo5Y/ET+hNyz+JnKDt8XLAdKs+AM0G5W0Vp1IrFI8N8= +github.com/hashicorp/vault/api/auth/approle v0.5.0 h1:a1TK6VGwYqSAfkmX4y4dJ4WBxMU5dStIZqScW4EPXR8= +github.com/hashicorp/vault/api/auth/approle v0.5.0/go.mod h1:CHOQIA1AZACfjTzHggmyfiOZ+xCSKNRFqe48FTCzH0k= +github.com/hashicorp/vault/api/auth/kubernetes v0.5.0 h1:CXO0fD7M3iCGovP/UApeHhPcH4paDFKcu7AjEXi94rI= +github.com/hashicorp/vault/api/auth/kubernetes v0.5.0/go.mod h1:afrElBIO9Q4sHFVuVWgNevG4uAs1bT2AZFA9aEiI608= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:VzM3TYHDgqPkettiP6I6q2jOeQFL4nrJM+UcAc4f6Fs= +github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1 h1:dQEHhTfi+bSIOSViQrKY9PqJvZenD6tFz+3lPzux58o= +github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1/go.mod h1:my+EVjOJLeQ9lUR9uVkxRvNNkhO2saSGIgzV8GZT9HY= +github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1 h1:bPR1KJK9pbSYuDPQzx5zXOT7Exj+y/K/8lpGU2KfzRc= +github.com/libopenstorage/secrets v0.0.0-20231011182615-5f4b25ceede1/go.mod h1:TB8PxROcwcNeaawFm45+XAj0lnZL2wRI3wTr/tZ3/bM= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.0/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/go-wordwrap v1.0.0/go.mod h1:ZXFpozHsX6DPmq2I0TCekCxypsnAUbP2oI0UX1GXzOo= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= +github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= +github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= +github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= +github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.0/go.mod h1:Luc4sArBICYCS8THh8v3i3i5CuSZO+RaQRaJoeNwomw= +github.com/onsi/ginkgo/v2 v2.6.0/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= +github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= +github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/openshift/api v0.0.0-20230816181854-a7ca92db022a h1:1bylAza0mFIchRCRPVY9qy62CxJE18fpjEAUSpIA5O4= +github.com/openshift/api v0.0.0-20230816181854-a7ca92db022a/go.mod h1:yimSGmjsI+XF1mr+AKBs2//fSXIOhhetHGbMlBEfXbs= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/portworx/dcos-secrets v0.0.0-20180616013705-8e8ec3f66611/go.mod h1:4hklRW/4DQpLqkcXcjtNprbH2tz/sJaNtqinfPWl/LA= +github.com/portworx/kvdb v0.0.0-20200929023115-b312c7519467/go.mod h1:Q8YyrNDvPp3DVF96BDcQuaC7fAYUCuUX+l58S7OnD2M= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4nPKWu0nJ5d+o= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= +github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= +github.com/spf13/afero v1.2.2/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk= +github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= +go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= +golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek= +golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY= +golang.org/x/exp v0.0.0-20191129062945-2f5052295587/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= +golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= +golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= +golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= +golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= +golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.1-0.20221206200815-1e63c2f08a10/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20220223155221-ee480838109b/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220309155454-6242fa91716a/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220411215720-9780585627b5/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.0.0-20220524215830-622c5d57e401/go.mod h1:DAh4E804XQdzx2j+YRIaUnCqCV2RuMz24cGBJ5QYIrc= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210831042530-f4d43177bf5e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191113191852-77e3bb0ad9e7/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191115202509-3a792d9c32b2/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191130070609-6e064ea0cf2d/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20191227053925-7b8e75db28f4/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200117161641-43d50277825c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200122220014-bf1340f18c4a/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200616133436-c1934b75d054/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.9.3 h1:Gn1I8+64MsuTb/HpH+LmQtNas23LhUVr3rYZ0eKuaMM= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M= +google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= +google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= +google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= +google.golang.org/api v0.55.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.56.0/go.mod h1:38yMfeP1kfjsl8isn0tliTjIb1rJXcQi4UXlbqivdVE= +google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdrMgI= +google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= +google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= +google.golang.org/api v0.67.0/go.mod h1:ShHKP8E60yPsKNw/w8w+VYaj9H6buA5UqDp8dhbQZ6g= +google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA= +google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8= +google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs= +google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69ljA= +google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw= +google.golang.org/api v0.83.0/go.mod h1:CNywQoj/AfhTw26ZWAa6LwOv+6WFxHmeLPZq2uncLZk= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= +google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191115194625-c23dd37a84c9/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20191230161307-f3c370f40bfb/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200115191322-ca5a22157cba/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= +google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= +google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210828152312-66f60bf46e71/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210831024726-fe130286e0e2/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210903162649-d08c68adba83/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210909211513-a8c4777a87af/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210924002016-3dee208752a0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211206160659-862468c7d6e0/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211208223120-3a66f561d7aa/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20211221195035-429b39de9b1c/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220126215142-9970aeb2e350/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220207164111-0872dc986b00/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc= +google.golang.org/genproto v0.0.0-20220218161850-94dd64e39d7c/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220310185008-1973136f34c6/go.mod h1:kGP+zUP2Ddo0ayMi4YuN7C3WZyJvGLZRh8Z5wnAqvEI= +google.golang.org/genproto v0.0.0-20220324131243-acbaeb5b85eb/go.mod h1:hAL49I2IFola2sVEjAn7MEwsja0xp51I0tlGAf9hz4E= +google.golang.org/genproto v0.0.0-20220407144326-9054f6ed7bac/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220413183235-5e96e2839df9/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220414192740-2d67ff6cf2b4/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220421151946-72621c1f0bd3/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220429170224-98d788798c3e/go.mod h1:8w6bsBMX6yCPbAVTeqQHvzxW0EIFigd5lZyahWgyfDo= +google.golang.org/genproto v0.0.0-20220505152158-f39f71e6c8f3/go.mod h1:RAyBrSAP7Fh3Nc84ghnVLDPuV51xc9agzmm4Ph6i0Q4= +google.golang.org/genproto v0.0.0-20220602131408-e326c6e8e9c8/go.mod h1:yKyY4AMRwFiC8yMMNaMi+RkCnjZJt9LoWuvhXjMs+To= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.46.2/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc v1.47.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= +gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +k8s.io/api v0.23.5/go.mod h1:Na4XuKng8PXJ2JsploYYrivXrINeTaycCGcYgF91Xm8= +k8s.io/api v0.26.0/go.mod h1:k6HDTaIFC8yn1i6pSClSqIwLABIcLV9l5Q4EcngKnQg= +k8s.io/api v0.28.3 h1:Gj1HtbSdB4P08C8rs9AR94MfSGpRhJgsS+GF9V26xMM= +k8s.io/api v0.28.3/go.mod h1:MRCV/jr1dW87/qJnZ57U5Pak65LGmQVkKTzf3AtKFHc= +k8s.io/apiextensions-apiserver v0.28.3 h1:Od7DEnhXHnHPZG+W9I97/fSQkVpVPQx2diy+2EtmY08= +k8s.io/apimachinery v0.23.5/go.mod h1:BEuFMMBaIbcOqVIJqNZJXGFTP4W6AycEpb5+m/97hrM= +k8s.io/apimachinery v0.26.0/go.mod h1:tnPmbONNJ7ByJNz9+n9kMjNP8ON+1qoAIIC70lztu74= +k8s.io/apimachinery v0.28.3 h1:B1wYx8txOaCQG0HmYF6nbpU8dg6HvA06x5tEffvOe7A= +k8s.io/apimachinery v0.28.3/go.mod h1:uQTKmIqs+rAYaq+DFaoD2X7pcjLOqbQX2AOiO0nIpb8= +k8s.io/client-go v0.23.5/go.mod h1:flkeinTO1CirYgzMPRWxUCnV0G4Fbu2vLhYCObnt/r4= +k8s.io/client-go v0.28.3 h1:2OqNb72ZuTZPKCl+4gTKvqao0AMOl9f3o2ijbAj3LI4= +k8s.io/client-go v0.28.3/go.mod h1:LTykbBp9gsA7SwqirlCXBWtK0guzfhpoW4qSm7i9dxo= +k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg= +k8s.io/component-base v0.28.3 h1:rDy68eHKxq/80RiMb2Ld/tbH8uAE75JdCqJyi6lXMzI= +k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0= +k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E= +k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE= +k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.4.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y= +k8s.io/klog/v2 v2.30.0/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.60.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.80.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/klog/v2 v2.100.1 h1:7WCHKK6K8fNhTqfBhISHQ97KrnJNFZMcQvKp7gP/tmg= +k8s.io/klog/v2 v2.100.1/go.mod h1:y1WjHnz7Dj687irZUWR/WLkLc5N1YHtjLdmgWjndZn0= +k8s.io/kube-openapi v0.0.0-20201113171705-d219536bb9fd/go.mod h1:WOJ3KddDSol4tAGcJo0Tvi+dK12EcqSLqcWsryKMpfM= +k8s.io/kube-openapi v0.0.0-20211115234752-e816edb12b65/go.mod h1:sX9MT8g7NVZM5lVL/j8QyCCJe8YSMW30QvGZWaCIDIk= +k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280/go.mod h1:+Axhij7bCpeqhklhUTe3xmOn6bWxolyZEeyaFpjGtl4= +k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f h1:eeEUOoGYWhOz7EyXqhlR2zHKNw2mNJ9vzJmub6YN6kk= +k8s.io/kube-openapi v0.0.0-20230905202853-d090da108d2f/go.mod h1:AsvuZPBlUDVuCdzJ87iajxtXuR9oktsTctW/R9wwouA= +k8s.io/utils v0.0.0-20210802155522-efc7438f0176/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20211116205334-6203023598ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= +k8s.io/utils v0.0.0-20221107191617-1a15be271d1d/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20221128185143-99ec85e7a448/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b h1:sgn3ZU783SCgtaSJjpcVVlRqd6GSnlTLKgpAAttJvpI= +k8s.io/utils v0.0.0-20230726121419-3b25d923346b/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= +rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +sigs.k8s.io/controller-runtime v0.16.3 h1:2TuvuokmfXvDUamSx1SuAOO3eTyye+47mJCigwG62c4= +sigs.k8s.io/controller-runtime v0.16.3/go.mod h1:j7bialYoSn142nv9sCOJmQgDXQXxnroFU4VnX/brVJ0= +sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6/go.mod h1:p4QtZmO4uMYipTQNzagwnNoseA6OxSUutVw05NhYDRs= +sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= +sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= +sigs.k8s.io/structured-merge-diff/v4 v4.0.2/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw= +sigs.k8s.io/structured-merge-diff/v4 v4.2.1/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= +sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0 h1:UZbZAZfX0wV2zr7YZorDz6GXROfDFj6LvqCRm4VUVKk= +sigs.k8s.io/structured-merge-diff/v4 v4.3.0/go.mod h1:N8hJocpFajUSSeSJ9bOZ77VzejKZaXsTtZo4/u7Io08= +sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= +sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= +sigs.k8s.io/yaml v1.4.0 h1:Mk1wCc2gy/F0THH0TAp1QYyJNzRm2KCLy3o5ASXVI5E= +sigs.k8s.io/yaml v1.4.0/go.mod h1:Ejl7/uTz7PSA4eKMyQCUTnhZYNmLIl+5c2lQPGR2BPY= diff --git a/pkg/daemon/ceph/client/fake/osd.go b/pkg/daemon/ceph/client/fake/osd.go index fb38052051ee..bac5408cbd8a 100644 --- a/pkg/daemon/ceph/client/fake/osd.go +++ b/pkg/daemon/ceph/client/fake/osd.go @@ -43,7 +43,7 @@ func OsdLsOutput(numOSDs int) string { // - osd3 - osd4 - osd5 // - osd6 - osd7 - osd8 func OsdTreeOutput(numNodes, numOSDsPerNode int) string { - // JSON output taken from Ceph Pacific + // JSON output rootFormat := ` { "id": -1, "name": "default", @@ -106,7 +106,6 @@ func OsdTreeOutput(numNodes, numOSDsPerNode int) string { // If returnOsdIds is empty, this returns a NOT ok-to-stop result. Otherwise, it returns an // ok-to-stop result. returnOsdIds should include queriedID if the result should be successful. func OsdOkToStopOutput(queriedID int, returnOsdIds []int) string { - // For Pacific and up (Pacific+) okTemplate := `{"ok_to_stop":true,"osds":[%s],"num_ok_pgs":132,"num_not_ok_pgs":0,"ok_become_degraded":["1.0","1.2","1.3"]}` notOkTemplate := `{"ok_to_stop":false,"osds":[%d],"num_ok_pgs":161,"num_not_ok_pgs":50,"bad_become_inactive":["1.0","1.3","1.a"],"ok_become_degraded":["1.2","1.4","1.5"]}` diff --git a/pkg/daemon/ceph/client/mgr.go b/pkg/daemon/ceph/client/mgr.go index 7dbc6003e6b5..d1dca7a19187 100755 --- a/pkg/daemon/ceph/client/mgr.go +++ b/pkg/daemon/ceph/client/mgr.go @@ -69,11 +69,8 @@ func MgrEnableModule(context *clusterd.Context, clusterInfo *ClusterInfo, name s retryCount := 5 var err error for i := 0; i < retryCount; i++ { - /* In Pacific the balancer is now on by default in upmap mode. - In earlier versions, the balancer was included in the ``always_on_modules`` list, but needed to be - turned on explicitly using the ``ceph balancer on`` command. */ if name == "balancer" { - logger.Debug("balancer module is already 'on' on pacific, doing nothing", name) + logger.Debug("balancer module is always 'on', doing nothing", name) return nil } else { err = enableModule(context, clusterInfo, name, force, "enable") diff --git a/pkg/daemon/ceph/client/mgr_test.go b/pkg/daemon/ceph/client/mgr_test.go index bbd08b2b1d5a..3df997dca823 100644 --- a/pkg/daemon/ceph/client/mgr_test.go +++ b/pkg/daemon/ceph/client/mgr_test.go @@ -21,7 +21,6 @@ import ( "github.com/pkg/errors" "github.com/rook/rook/pkg/clusterd" - "github.com/rook/rook/pkg/operator/ceph/version" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" ) @@ -60,16 +59,9 @@ func TestEnableModuleRetries(t *testing.T) { _ = MgrEnableModule(&clusterd.Context{Executor: executor}, clusterInfo, "pg_autoscaler", false) assert.Equal(t, 0, moduleEnableRetries) - // Balancer not on Ceph Pacific - moduleEnableRetries = 0 + // Balancer skipped _ = MgrEnableModule(&clusterd.Context{Executor: executor}, clusterInfo, "balancer", false) assert.Equal(t, 0, moduleEnableRetries) - - // Balancer skipped on Pacific - clusterInfo.CephVersion = version.Pacific - _ = MgrEnableModule(&clusterd.Context{Executor: executor}, clusterInfo, "balancer", false) - assert.Equal(t, 0, moduleEnableRetries) - } func TestEnableModule(t *testing.T) { diff --git a/pkg/daemon/ceph/client/osd_test.go b/pkg/daemon/ceph/client/osd_test.go index 6a4a4270bb8b..b6a2c77f0a13 100644 --- a/pkg/daemon/ceph/client/osd_test.go +++ b/pkg/daemon/ceph/client/osd_test.go @@ -167,9 +167,9 @@ func TestOSDOkToStop(t *testing.T) { seenArgs = []string{} } - t.Run("pacific output ok to stop", func(t *testing.T) { + t.Run("output ok to stop", func(t *testing.T) { doSetup() - clusterInfo.CephVersion = version.Pacific + clusterInfo.CephVersion = version.Reef returnString = fake.OsdOkToStopOutput(1, []int{1, 2}) returnOkResult = true osds, err := OSDOkToStop(context, clusterInfo, 1, 2) @@ -179,9 +179,9 @@ func TestOSDOkToStop(t *testing.T) { assert.Equal(t, "--max=2", seenArgs[3]) }) - t.Run("pacific output not ok to stop", func(t *testing.T) { + t.Run("output not ok to stop", func(t *testing.T) { doSetup() - clusterInfo.CephVersion = version.Pacific + clusterInfo.CephVersion = version.Reef returnString = fake.OsdOkToStopOutput(3, []int{}) returnOkResult = false _, err := OSDOkToStop(context, clusterInfo, 3, 5) @@ -190,9 +190,9 @@ func TestOSDOkToStop(t *testing.T) { assert.Equal(t, "--max=5", seenArgs[3]) }) - t.Run("pacific handles maxReturned=0", func(t *testing.T) { + t.Run("handle maxReturned=0", func(t *testing.T) { doSetup() - clusterInfo.CephVersion = version.Pacific + clusterInfo.CephVersion = version.Reef returnString = fake.OsdOkToStopOutput(4, []int{4, 8}) returnOkResult = true osds, err := OSDOkToStop(context, clusterInfo, 4, 0) diff --git a/pkg/daemon/ceph/client/pool.go b/pkg/daemon/ceph/client/pool.go index e8ad144fdc1d..afac5c61d73a 100644 --- a/pkg/daemon/ceph/client/pool.go +++ b/pkg/daemon/ceph/client/pool.go @@ -471,21 +471,21 @@ func createReplicatedPoolForApp(context *clusterd.Context, clusterInfo *ClusterI logger.Infof("reconciling replicated pool %s succeeded", pool.Name) - if checkFailureDomain { - if err = ensureFailureDomain(context, clusterInfo, clusterSpec, pool); err != nil { + if checkFailureDomain || pool.PoolSpec.DeviceClass != "" { + if err = updatePoolCrushRule(context, clusterInfo, clusterSpec, pool); err != nil { return nil } } return nil } -func ensureFailureDomain(context *clusterd.Context, clusterInfo *ClusterInfo, clusterSpec *cephv1.ClusterSpec, pool cephv1.NamedPoolSpec) error { - if pool.FailureDomain == "" { - logger.Debugf("skipping check for failure domain on pool %q as it is not specified", pool.Name) +func updatePoolCrushRule(context *clusterd.Context, clusterInfo *ClusterInfo, clusterSpec *cephv1.ClusterSpec, pool cephv1.NamedPoolSpec) error { + if pool.FailureDomain == "" && pool.DeviceClass == "" { + logger.Debugf("skipping check for failure domain and deviceClass on pool %q as it is not specified", pool.Name) return nil } - logger.Debugf("checking that pool %q has the failure domain %q", pool.Name, pool.FailureDomain) + logger.Debugf("checking that pool %q has the failure domain %q and deviceClass %q", pool.Name, pool.FailureDomain, pool.DeviceClass) details, err := GetPoolDetails(context, clusterInfo, pool.Name) if err != nil { return errors.Wrapf(err, "failed to get pool %q details", pool.Name) @@ -496,17 +496,24 @@ func ensureFailureDomain(context *clusterd.Context, clusterInfo *ClusterInfo, cl if err != nil { return errors.Wrapf(err, "failed to get crush rule %q", details.CrushRule) } - currentFailureDomain := extractFailureDomain(rule) - if currentFailureDomain == pool.FailureDomain { - logger.Debugf("pool %q has the expected failure domain %q", pool.Name, pool.FailureDomain) + currentFailureDomain, currentDeviceClass := extractPoolDetails(rule) + if currentFailureDomain == pool.FailureDomain && currentDeviceClass == pool.DeviceClass { + logger.Infof("pool %q has the expected failure domain %q and deviceClass %q", pool.Name, pool.FailureDomain, pool.DeviceClass) return nil } - if currentFailureDomain == "" { - logger.Warningf("failure domain not found for crush rule %q, proceeding to create a new crush rule", details.CrushRule) + + if currentFailureDomain != pool.FailureDomain { + logger.Infof("creating a new crush rule for changed failure domain on crush rule %q", details.CrushRule) + } + if currentDeviceClass != pool.DeviceClass { + logger.Infof("creating a new crush rule for changed deviceClass on crush rule %q", details.CrushRule) } // Use a crush rule name that is unique to the desired failure domain crushRuleName := fmt.Sprintf("%s_%s", pool.Name, pool.FailureDomain) + if pool.DeviceClass != "" { + crushRuleName = fmt.Sprintf("%s_%s_%s", pool.Name, pool.FailureDomain, pool.DeviceClass) + } logger.Infof("updating pool %q failure domain from %q to %q with new crush rule %q", pool.Name, currentFailureDomain, pool.FailureDomain, crushRuleName) logger.Infof("crush rule %q will no longer be used by pool %q", details.CrushRule, pool.Name) @@ -524,12 +531,17 @@ func ensureFailureDomain(context *clusterd.Context, clusterInfo *ClusterInfo, cl return nil } -func extractFailureDomain(rule ruleSpec) string { +func extractPoolDetails(rule ruleSpec) (string, string) { // find the failure domain in the crush rule, which is the first step where the // "type" property is set + + var failureDomain, deviceClass string for i, step := range rule.Steps { if step.Type != "" { - return step.Type + failureDomain = step.Type + } + if step.ItemName != "" { + deviceClass = step.ItemName } // We expect the rule to be found by the second step, or else it is a more // complex rule that would not be supported for updating the failure domain @@ -537,7 +549,7 @@ func extractFailureDomain(rule ruleSpec) string { break } } - return "" + return failureDomain, deviceClass } func setCrushRule(context *clusterd.Context, clusterInfo *ClusterInfo, poolName, crushRule string) error { diff --git a/pkg/daemon/ceph/client/pool_test.go b/pkg/daemon/ceph/client/pool_test.go index bcbaf820d163..7b101d14ebd0 100644 --- a/pkg/daemon/ceph/client/pool_test.go +++ b/pkg/daemon/ceph/client/pool_test.go @@ -281,7 +281,7 @@ func TestUpdateFailureDomain(t *testing.T) { }, } clusterSpec := &cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{}} - err := ensureFailureDomain(context, AdminTestClusterInfo("mycluster"), clusterSpec, p) + err := updatePoolCrushRule(context, AdminTestClusterInfo("mycluster"), clusterSpec, p) assert.NoError(t, err) assert.Equal(t, "", newCrushRule) }) @@ -295,7 +295,7 @@ func TestUpdateFailureDomain(t *testing.T) { }, } clusterSpec := &cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{}} - err := ensureFailureDomain(context, AdminTestClusterInfo("mycluster"), clusterSpec, p) + err := updatePoolCrushRule(context, AdminTestClusterInfo("mycluster"), clusterSpec, p) assert.NoError(t, err) assert.Equal(t, "", newCrushRule) }) @@ -309,29 +309,30 @@ func TestUpdateFailureDomain(t *testing.T) { }, } clusterSpec := &cephv1.ClusterSpec{Storage: cephv1.StorageScopeSpec{}} - err := ensureFailureDomain(context, AdminTestClusterInfo("mycluster"), clusterSpec, p) + err := updatePoolCrushRule(context, AdminTestClusterInfo("mycluster"), clusterSpec, p) assert.NoError(t, err) assert.Equal(t, "mypool_zone", newCrushRule) }) } -func TestExtractFailureDomain(t *testing.T) { +func TestExtractPoolDetails(t *testing.T) { t.Run("complex crush rule skipped", func(t *testing.T) { rule := ruleSpec{Steps: []stepSpec{ {Type: ""}, {Type: ""}, {Type: "zone"}, }} - failureDomain := extractFailureDomain(rule) + failureDomain, _ := extractPoolDetails(rule) assert.Equal(t, "", failureDomain) }) t.Run("valid crush rule", func(t *testing.T) { rule := ruleSpec{Steps: []stepSpec{ {Type: ""}, - {Type: "zone"}, + {Type: "zone", ItemName: "ssd"}, }} - failureDomain := extractFailureDomain(rule) + failureDomain, deviceClass := extractPoolDetails(rule) assert.Equal(t, "zone", failureDomain) + assert.Equal(t, "ssd", deviceClass) }) } diff --git a/pkg/daemon/ceph/client/upgrade.go b/pkg/daemon/ceph/client/upgrade.go index 0235eee803e2..9fea46170209 100644 --- a/pkg/daemon/ceph/client/upgrade.go +++ b/pkg/daemon/ceph/client/upgrade.go @@ -214,12 +214,12 @@ func okToContinueMDSDaemon(context *clusterd.Context, clusterInfo *ClusterInfo, // Assume the following: // // "mon": { -// "ceph version 16.2.5 (cbff874f9007f1869bfd3821b7e33b2a6ffd4988) pacific (stable)": 2, -// "ceph version 17.2.0 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) quincy (stable)": 1 +// "ceph version 18.2.5 (cbff874f9007f1869bfd3821b7e33b2a6ffd4988) reef (stable)": 2, +// "ceph version 17.2.6 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) quincy (stable)": 1 // } // -// In the case we will pick: "ceph version 16.2.5 (cbff874f9007f1869bfd3821b7e33b2a6ffd4988) pacific (stable)": 2, -// And eventually return 16.2.5 +// In the case we will pick: "ceph version 18.2.5 (cbff874f9007f1869bfd3821b7e33b2a6ffd4988) reef (stable)": 2, +// And eventually return 18.2.5 func LeastUptodateDaemonVersion(context *clusterd.Context, clusterInfo *ClusterInfo, daemonType string) (cephver.CephVersion, error) { var r map[string]int var vv cephver.CephVersion diff --git a/pkg/daemon/ceph/client/upgrade_test.go b/pkg/daemon/ceph/client/upgrade_test.go index 0273fb179975..6630fd196886 100644 --- a/pkg/daemon/ceph/client/upgrade_test.go +++ b/pkg/daemon/ceph/client/upgrade_test.go @@ -128,8 +128,8 @@ func TestDaemonMapEntry(t *testing.T) { dummyVersionsRaw := []byte(` { "mon": { - "ceph version 16.2.5 (cbff874f9007f1869bfd3821b7e33b2a6ffd4988) pacific (stable)": 1, - "ceph version 17.2.0 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) quincy (stable)": 2 + "ceph version 18.2.5 (cbff874f9007f1869bfd3821b7e33b2a6ffd4988) reef (stable)": 1, + "ceph version 17.2.7 (3a54b2b6d167d4a2a19e003a705696d4fe619afc) quincy (stable)": 2 } }`) diff --git a/pkg/daemon/ceph/osd/daemon.go b/pkg/daemon/ceph/osd/daemon.go index 9baa9dc7e5a8..e8e9b68a1756 100644 --- a/pkg/daemon/ceph/osd/daemon.go +++ b/pkg/daemon/ceph/osd/daemon.go @@ -446,6 +446,11 @@ func getAvailableDevices(context *clusterd.Context, agent *OsdAgent) (*DeviceOsd } logger.Infof("device %q is available.", device.Name) + if device.Type == sys.PartType && agent.storeConfig.EncryptedDevice { + logger.Infof("partition %q is not picked because encrypted OSD on partition is not allowed", device.Name) + continue + } + var deviceInfo *DeviceOsdIDEntry if agent.metadataDevice != "" && agent.metadataDevice == device.Name { // current device is desired as the metadata device diff --git a/pkg/daemon/ceph/osd/volume.go b/pkg/daemon/ceph/osd/volume.go index e6ff5c37cfac..b127f265d0b7 100644 --- a/pkg/daemon/ceph/osd/volume.go +++ b/pkg/daemon/ceph/osd/volume.go @@ -415,6 +415,12 @@ func (a *OsdAgent) allowRawMode(context *clusterd.Context) (bool, error) { // by default assume raw mode allowRawMode := true + // ceph-volume raw mode does not support encryption yet + if a.storeConfig.EncryptedDevice { + logger.Debug("won't use raw mode since encryption is enabled") + allowRawMode = false + } + // ceph-volume raw mode does not support more than one OSD per disk osdsPerDeviceCountString := sanitizeOSDsPerDevice(a.storeConfig.OSDsPerDevice) osdsPerDeviceCount, err := strconv.Atoi(osdsPerDeviceCountString) diff --git a/pkg/daemon/ceph/osd/volume_test.go b/pkg/daemon/ceph/osd/volume_test.go index 2fa3663e837f..95366f040b53 100644 --- a/pkg/daemon/ceph/osd/volume_test.go +++ b/pkg/daemon/ceph/osd/volume_test.go @@ -1599,7 +1599,7 @@ func TestAllowRawMode(t *testing.T) { wantErr bool }{ {"raw simple scenario supported", fields{"", config.StoreConfig{}}, args{&clusterd.Context{}, false}, true, false}, - {"raw encrypted scenario supported", fields{"", config.StoreConfig{EncryptedDevice: true}}, args{&clusterd.Context{}, false}, true, false}, + {"lvm complex scenario not supported: encrypted", fields{"", config.StoreConfig{EncryptedDevice: true}}, args{&clusterd.Context{}, false}, false, false}, {"lvm complex scenario not supported: osd per device > 1", fields{"", config.StoreConfig{OSDsPerDevice: 2}}, args{&clusterd.Context{}, false}, false, false}, {"lvm complex scenario not supported: metadata dev", fields{"/dev/sdb", config.StoreConfig{}}, args{&clusterd.Context{}, false}, false, false}, {"lvm complex scenario not supported: metadata dev", fields{"/dev/sdb", config.StoreConfig{}}, args{&clusterd.Context{}, false}, false, false}, diff --git a/pkg/daemon/multus/config.go b/pkg/daemon/multus/config.go index 9ec52a2c30bf..e493c6971b6d 100644 --- a/pkg/daemon/multus/config.go +++ b/pkg/daemon/multus/config.go @@ -46,6 +46,8 @@ var ( DefaultValidationResourceTimeout = 3 * time.Minute + DefaultValidationFlakyThreshold = 30 * time.Second + DefaultStorageNodeLabelKey = "storage-node" DefaultStorageNodeLabelValue = "true" @@ -72,6 +74,7 @@ type ValidationTestConfig struct { PublicNetwork string `yaml:"publicNetwork"` ClusterNetwork string `yaml:"clusterNetwork"` ResourceTimeout time.Duration `yaml:"resourceTimeout"` + FlakyThreshold time.Duration `yaml:"flakyThreshold"` NginxImage string `yaml:"nginxImage"` NodeTypes map[string]NodeConfig `yaml:"nodeTypes"` } @@ -113,6 +116,7 @@ func NewDefaultValidationTestConfig() *ValidationTestConfig { return &ValidationTestConfig{ Namespace: DefaultValidationNamespace, ResourceTimeout: DefaultValidationResourceTimeout, + FlakyThreshold: DefaultValidationFlakyThreshold, NginxImage: DefaultValidationNginxImage, NodeTypes: map[string]NodeConfig{ DefaultValidationNodeType: { @@ -209,6 +213,9 @@ func (c *ValidationTestConfig) Validate() error { if c.ResourceTimeout < 1*time.Minute { errs = append(errs, "resourceTimeout must be at least one minute (two or more are recommended)") } + if c.FlakyThreshold < 5*time.Second { + errs = append(errs, "flaky threshold must be at least 5 seconds") + } if c.NginxImage == "" { errs = append(errs, "nginxImage must be specified") } @@ -261,6 +268,7 @@ func NewDedicatedStorageNodesValidationTestConfig() *ValidationTestConfig { return &ValidationTestConfig{ Namespace: DefaultValidationNamespace, ResourceTimeout: DefaultValidationResourceTimeout, + FlakyThreshold: DefaultValidationFlakyThreshold, NginxImage: DefaultValidationNginxImage, NodeTypes: map[string]NodeConfig{ DedicatedStorageNodeType: dedicatedStorageNodeConfig, @@ -277,6 +285,7 @@ func NewArbiterValidationTestConfig() *ValidationTestConfig { return &ValidationTestConfig{ Namespace: DefaultValidationNamespace, ResourceTimeout: DefaultValidationResourceTimeout, + FlakyThreshold: DefaultValidationFlakyThreshold, NginxImage: DefaultValidationNginxImage, NodeTypes: map[string]NodeConfig{ DedicatedStorageNodeType: dedicatedStorageNodeConfig, diff --git a/pkg/daemon/multus/config.yaml b/pkg/daemon/multus/config.yaml index 32de65f0559c..e38fea5c8980 100644 --- a/pkg/daemon/multus/config.yaml +++ b/pkg/daemon/multus/config.yaml @@ -16,6 +16,19 @@ clusterNetwork: "{{ .ClusterNetwork }}" # This should be at least 1 minute. 2 minutes or more is recommended. resourceTimeout: "{{ .ResourceTimeout }}" +# This is the time window in which validation clients are all expected to become "Ready" together. +# Validation clients are all started at approximately the same time, and they should all stabilize +# at approximately the same time. Once the first validation client becomes "Ready," the tool checks +# that all of the remaining clients become "Ready" before this threshold duration elapses. +# In networks that have connectivity issues, limited bandwidth, or high latency, clients will contend +# for network traffic with each other, causing some clients to randomly fail and become "Ready" later +# than others. These randomly-failing clients are considered "flaky." Adjust this value to reflect +# expectations for the underlying network. For fast and reliable networks, this can be set to a smaller +# value. For networks that are intended to be slow, this can be set to a larger value. Additionally, for +# very large Kubernetes clusters, it may take longer for all clients to start, and it therefore may take +# longer for all clients to become "Ready"; in that case, this value can be set slightly higher. +flakyThreshold: "{{ .FlakyThreshold }}" + # The Nginx image which will be used for the web server and clients. nginxImage: "{{ .NginxImage }}" diff --git a/pkg/daemon/multus/config_test.go b/pkg/daemon/multus/config_test.go index fdc4099c2d97..0e4387017eae 100644 --- a/pkg/daemon/multus/config_test.go +++ b/pkg/daemon/multus/config_test.go @@ -40,6 +40,7 @@ func TestValidationTestConfig_YAML(t *testing.T) { PublicNetwork: "my-pub", ClusterNetwork: "my-priv", ResourceTimeout: 2 * time.Minute, + FlakyThreshold: 30 * time.Second, NginxImage: "myorg/nginx:latest", NodeTypes: map[string]NodeConfig{ "osdOnlyNodes": { diff --git a/pkg/daemon/multus/validation.go b/pkg/daemon/multus/validation.go index 5a72cf0b0497..d2676e2cc92b 100644 --- a/pkg/daemon/multus/validation.go +++ b/pkg/daemon/multus/validation.go @@ -313,7 +313,7 @@ func (s *verifyAllClientsReadyState) Run(ctx context.Context, vsm *validationSta suggestionsOnSuccess := []string{} if s.suggestFlaky { suggestionsOnSuccess = append(suggestionsOnSuccess, - fmt.Sprintf("not all clients became ready within %s; %s", flakyThreshold.String(), flakyNetworkSuggestion)) + fmt.Sprintf("not all clients became ready within %s; %s", vsm.vt.FlakyThreshold.String(), flakyNetworkSuggestion)) } vsm.Exit() // DONE! return suggestionsOnSuccess, nil @@ -321,8 +321,6 @@ func (s *verifyAllClientsReadyState) Run(ctx context.Context, vsm *validationSta // clients should all become ready within a pretty short amount of time since they all should start // pretty simultaneously -// TODO: allow tuning this -var flakyThreshold = 20 * time.Second func (s *verifyAllClientsReadyState) checkIfFlaky(vsm *validationStateMachine, numReady int) { if s.suggestFlaky { @@ -338,9 +336,9 @@ func (s *verifyAllClientsReadyState) checkIfFlaky(vsm *validationStateMachine, n if !s.timeClientsStartedBecomingReady.IsZero() { // check to see how long it took since clients first started becoming ready. if the time is // longer than the flaky threshold, warn the user, and record that the network is flaky - if time.Since(s.timeClientsStartedBecomingReady) > flakyThreshold { + if time.Since(s.timeClientsStartedBecomingReady) > vsm.vt.FlakyThreshold { vsm.vt.Logger.Warningf( - "network seems flaky; the time since clients started becoming ready until now is greater than %s", flakyThreshold.String()) + "network seems flaky; the time since clients started becoming ready until now is greater than %s", vsm.vt.FlakyThreshold.String()) s.suggestFlaky = true } } diff --git a/pkg/operator/ceph/cluster/cluster.go b/pkg/operator/ceph/cluster/cluster.go index 9ba83f9f6fd5..841e0b1e58fa 100755 --- a/pkg/operator/ceph/cluster/cluster.go +++ b/pkg/operator/ceph/cluster/cluster.go @@ -444,32 +444,6 @@ func (c *cluster) replaceDefaultCrushMap(newRoot string) (err error) { // preMonStartupActions is a collection of actions to run before the monitors are reconciled. func (c *cluster) preMonStartupActions(cephVersion cephver.CephVersion) error { - // Disable the mds sanity checks for the mons due to a ceph upgrade issue - // for the mds to Pacific if 16.2.7 or greater. We keep it more general for any - // Pacific upgrade greater than 16.2.7 in case they skip upgrading directly to 16.2.7. - if c.isUpgrade && cephVersion.IsPacific() && cephVersion.IsAtLeast(cephver.CephVersion{Major: 16, Minor: 2, Extra: 7}) { - if err := c.skipMDSSanityChecks(true); err != nil { - // If there is an error, just print it and continue. Likely there is not a - // negative consequence of continuing since several complex conditions must exist to hit - // the upgrade issue where the sanity checks need to be disabled. - logger.Warningf("failed to disable the mon_mds_skip_sanity. %v", err) - } - } - return nil -} - -func (c *cluster) skipMDSSanityChecks(skip bool) error { - // In a running cluster disable the mds skip sanity setting during upgrades. - monStore := config.GetMonStore(c.context, c.ClusterInfo) - if skip { - if err := monStore.Set("mon", "mon_mds_skip_sanity", "1"); err != nil { - return err - } - } else { - if err := monStore.Delete("mon", "mon_mds_skip_sanity"); err != nil { - return err - } - } return nil } @@ -489,15 +463,6 @@ func (c *cluster) postMonStartupActions() error { return errors.Wrap(err, "failed to create crash collector kubernetes secret") } - // Always ensure the skip mds sanity checks setting is cleared, for all Pacific deployments - if c.ClusterInfo.CephVersion.IsPacific() { - if err := c.skipMDSSanityChecks(false); err != nil { - // If there is an error, just print it and continue. We can just try again - // at the next reconcile. - logger.Warningf("failed to re-enable the mon_mds_skip_sanity. %v", err) - } - } - if err := c.configureMsgr2(); err != nil { return errors.Wrap(err, "failed to configure msgr2") } diff --git a/pkg/operator/ceph/cluster/cluster_test.go b/pkg/operator/ceph/cluster/cluster_test.go index fed3ae2ade1a..282476ebea25 100644 --- a/pkg/operator/ceph/cluster/cluster_test.go +++ b/pkg/operator/ceph/cluster/cluster_test.go @@ -82,80 +82,6 @@ func TestPreClusterStartValidation(t *testing.T) { } } -func TestPreMonChecks(t *testing.T) { - executor := &exectest.MockExecutor{} - context := &clusterd.Context{Executor: executor} - setSkipSanity := false - unsetSkipSanity := false - executor.MockExecuteCommandWithTimeout = func(timeout time.Duration, command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) - if args[0] == "config" { - if args[1] == "set" { - setSkipSanity = true - assert.Equal(t, "mon", args[2]) - assert.Equal(t, "mon_mds_skip_sanity", args[3]) - assert.Equal(t, "1", args[4]) - return "", nil - } - if args[1] == "rm" { - unsetSkipSanity = true - assert.Equal(t, "mon", args[2]) - assert.Equal(t, "mon_mds_skip_sanity", args[3]) - return "", nil - } - } - return "", errors.Errorf("unexpected ceph command %q", args) - } - c := cluster{context: context, ClusterInfo: cephclient.AdminTestClusterInfo("cluster")} - - t.Run("no upgrade", func(t *testing.T) { - v := cephver.CephVersion{Major: 16, Minor: 2, Extra: 7} - c.isUpgrade = false - err := c.preMonStartupActions(v) - assert.NoError(t, err) - assert.False(t, setSkipSanity) - assert.False(t, unsetSkipSanity) - }) - - t.Run("upgrade below version", func(t *testing.T) { - setSkipSanity = false - unsetSkipSanity = false - v := cephver.CephVersion{Major: 16, Minor: 2, Extra: 6} - c.isUpgrade = true - err := c.preMonStartupActions(v) - assert.NoError(t, err) - assert.False(t, setSkipSanity) - assert.False(t, unsetSkipSanity) - }) - - t.Run("upgrade to applicable version", func(t *testing.T) { - setSkipSanity = false - unsetSkipSanity = false - v := cephver.CephVersion{Major: 16, Minor: 2, Extra: 7} - c.isUpgrade = true - err := c.preMonStartupActions(v) - assert.NoError(t, err) - assert.True(t, setSkipSanity) - assert.False(t, unsetSkipSanity) - - // This will be called during the post mon checks - err = c.skipMDSSanityChecks(false) - assert.NoError(t, err) - assert.True(t, unsetSkipSanity) - }) - - t.Run("upgrade to quincy", func(t *testing.T) { - setSkipSanity = false - unsetSkipSanity = false - v := cephver.CephVersion{Major: 17, Minor: 2, Extra: 0} - c.isUpgrade = true - err := c.preMonStartupActions(v) - assert.NoError(t, err) - assert.False(t, setSkipSanity) - assert.False(t, unsetSkipSanity) - }) -} - func TestConfigureMsgr2(t *testing.T) { type fields struct { expectedGlobalConfigSettings map[string]string diff --git a/pkg/operator/ceph/cluster/mgr/mgr.go b/pkg/operator/ceph/cluster/mgr/mgr.go index 6c1ac5b2cd02..aa55ef0dedd5 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr.go +++ b/pkg/operator/ceph/cluster/mgr/mgr.go @@ -530,6 +530,10 @@ func (c *Cluster) EnableServiceMonitor() error { if c.spec.External.Enable { serviceMonitor.Spec.Endpoints[0].Port = controller.ServiceExternalMetricName } + if c.spec.Monitoring.Interval != nil { + duration := c.spec.Monitoring.Interval.Duration.String() + serviceMonitor.Spec.Endpoints[0].Interval = monitoringv1.Duration(duration) + } err := c.clusterInfo.OwnerInfo.SetControllerReference(serviceMonitor) if err != nil { return errors.Wrapf(err, "failed to set owner reference to service monitor %q", serviceMonitor.Name) diff --git a/pkg/operator/ceph/cluster/mgr/mgr_test.go b/pkg/operator/ceph/cluster/mgr/mgr_test.go index b161bd1619a8..056b88673c93 100644 --- a/pkg/operator/ceph/cluster/mgr/mgr_test.go +++ b/pkg/operator/ceph/cluster/mgr/mgr_test.go @@ -23,7 +23,6 @@ import ( "testing" "time" - "github.com/pkg/errors" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/client/clientset/versioned/scheme" @@ -416,37 +415,6 @@ func TestApplyMonitoringLabels(t *testing.T) { assert.Nil(t, sm.Spec.Endpoints[0].RelabelConfigs) } -func TestCluster_enableBalancerModule(t *testing.T) { - c := &Cluster{ - context: &clusterd.Context{Executor: &exectest.MockExecutor{}, Clientset: testop.New(t, 3)}, - clusterInfo: cephclient.AdminTestClusterInfo("mycluster"), - } - - t.Run("on pacific we configure the balancer ONLY and don't set a mode", func(t *testing.T) { - c.clusterInfo.CephVersion = cephver.Pacific - executor := &exectest.MockExecutor{ - MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { - logger.Infof("Command: %s %v", command, args) - if command == "ceph" { - if args[0] == "osd" && args[1] == "set-require-min-compat-client" { - return "", nil - } - if args[0] == "balancer" && args[1] == "mode" { - return "", errors.New("balancer mode must not be set") - } - if args[0] == "balancer" && args[1] == "on" { - return "", nil - } - } - return "", errors.New("unknown command") - }, - } - c.context.Executor = executor - err := c.enableBalancerModule() - assert.NoError(t, err) - }) -} - func TestCluster_configurePrometheusModule(t *testing.T) { modulesEnabled := 0 modulesDisabled := 0 diff --git a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go index 4163010f809f..58c5923aa01c 100644 --- a/pkg/operator/ceph/cluster/mgr/orchestrator_test.go +++ b/pkg/operator/ceph/cluster/mgr/orchestrator_test.go @@ -86,7 +86,7 @@ func TestOrchestratorModules(t *testing.T) { assert.True(t, rookModuleEnabled) assert.True(t, rookBackendSet) - c.clusterInfo.CephVersion = cephver.Pacific + c.clusterInfo.CephVersion = cephver.Reef err = c.setRookOrchestratorBackend() assert.NoError(t, err) executor.MockExecuteCommandWithTimeout = func(timeout time.Duration, command string, args ...string) (string, error) { diff --git a/pkg/operator/ceph/cluster/osd/integration_test.go b/pkg/operator/ceph/cluster/osd/integration_test.go index 7918d239d556..252f69cc847b 100644 --- a/pkg/operator/ceph/cluster/osd/integration_test.go +++ b/pkg/operator/ceph/cluster/osd/integration_test.go @@ -188,7 +188,7 @@ func testOSDIntegration(t *testing.T) { clientset.PrependReactor("*", "deployments", deploymentReactor) clusterInfo := cephclient.NewClusterInfo(namespace, clusterName) - clusterInfo.CephVersion = cephver.Pacific + clusterInfo.CephVersion = cephver.Reef clusterInfo.SetName("mycluster") clusterInfo.OwnerInfo = cephclient.NewMinimumOwnerInfo(t) clusterInfo.Context = ctx diff --git a/pkg/operator/ceph/cluster/osd/update_test.go b/pkg/operator/ceph/cluster/osd/update_test.go index 3ab5305b60cf..61ca4425b5be 100644 --- a/pkg/operator/ceph/cluster/osd/update_test.go +++ b/pkg/operator/ceph/cluster/osd/update_test.go @@ -101,7 +101,7 @@ func Test_updateExistingOSDs(t *testing.T) { } clusterInfo := &cephclient.ClusterInfo{ Namespace: namespace, - CephVersion: cephver.Pacific, + CephVersion: cephver.Reef, Context: context.TODO(), } clusterInfo.SetName("mycluster") diff --git a/pkg/operator/ceph/cluster/rbd/controller_test.go b/pkg/operator/ceph/cluster/rbd/controller_test.go index f73912456bce..1bfbf3c4f3f4 100644 --- a/pkg/operator/ceph/cluster/rbd/controller_test.go +++ b/pkg/operator/ceph/cluster/rbd/controller_test.go @@ -64,7 +64,7 @@ func TestCephRBDMirrorController(t *testing.T) { os.Setenv("ROOK_LOG_LEVEL", "DEBUG") currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { - return &version.Pacific, &version.Pacific, nil + return &version.Reef, &version.Reef, nil } // An rbd-mirror resource with metadata and spec. diff --git a/pkg/operator/ceph/cluster/version_test.go b/pkg/operator/ceph/cluster/version_test.go index e0df9bb43fbd..5f72340834b9 100755 --- a/pkg/operator/ceph/cluster/version_test.go +++ b/pkg/operator/ceph/cluster/version_test.go @@ -160,10 +160,8 @@ func TestMinVersion(t *testing.T) { v = &cephver.CephVersion{Major: 15, Minor: 2, Extra: 11} assert.Error(t, c.validateCephVersion(v)) - // All versions at least 16.2.0 are valid - v = &cephver.CephVersion{Major: 16, Minor: 2, Extra: 0} - assert.NoError(t, c.validateCephVersion(v)) - v = &cephver.CephVersion{Major: 17} + // All versions at least 17.2.0 are valid + v = &cephver.CephVersion{Major: 17, Minor: 2} assert.NoError(t, c.validateCephVersion(v)) v = &cephver.CephVersion{Major: 18} assert.NoError(t, c.validateCephVersion(v)) @@ -174,13 +172,9 @@ func TestSupportedVersion(t *testing.T) { c.ClusterInfo = &client.ClusterInfo{Context: context.TODO()} // lower version is not supported - v := &cephver.CephVersion{Major: 15, Minor: 2, Extra: 5} + v := &cephver.CephVersion{Major: 16, Minor: 2, Extra: 7} assert.Error(t, c.validateCephVersion(v)) - // Pacific is supported - v = &cephver.CephVersion{Major: 16, Minor: 2, Extra: 0} - assert.NoError(t, c.validateCephVersion(v)) - // Quincy is supported v = &cephver.CephVersion{Major: 17, Minor: 2, Extra: 0} assert.NoError(t, c.validateCephVersion(v)) diff --git a/pkg/operator/ceph/config/config.go b/pkg/operator/ceph/config/config.go index 5e11bd6a0a0f..434585b5ef50 100644 --- a/pkg/operator/ceph/config/config.go +++ b/pkg/operator/ceph/config/config.go @@ -29,7 +29,6 @@ import ( cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/operator/ceph/version" ) var logger = capnslog.NewPackageLogger("github.com/rook/rook", "op-config") @@ -148,11 +147,6 @@ func SetOrRemoveDefaultConfigs( } func DisableInsecureGlobalID(context *clusterd.Context, clusterInfo *cephclient.ClusterInfo) { - if !canDisableInsecureGlobalID(clusterInfo) { - logger.Infof("cannot disable insecure global id on ceph version %v", clusterInfo.CephVersion.String()) - return - } - monStore := GetMonStore(context, clusterInfo) if err := monStore.Set("mon", "auth_allow_insecure_global_id_reclaim", "false"); err != nil { logger.Warningf("failed to disable the insecure global ID. %v", err) @@ -160,14 +154,3 @@ func DisableInsecureGlobalID(context *clusterd.Context, clusterInfo *cephclient. logger.Info("insecure global ID is now disabled") } } - -func canDisableInsecureGlobalID(clusterInfo *cephclient.ClusterInfo) bool { - cephver := clusterInfo.CephVersion - if cephver.IsAtLeastQuincy() { - return true - } - if cephver.IsPacific() && cephver.IsAtLeast(version.CephVersion{Major: 16, Minor: 2, Extra: 1}) { - return true - } - return false -} diff --git a/pkg/operator/ceph/config/config_test.go b/pkg/operator/ceph/config/config_test.go index b371f29916d1..89a4b477f0d9 100644 --- a/pkg/operator/ceph/config/config_test.go +++ b/pkg/operator/ceph/config/config_test.go @@ -19,8 +19,6 @@ package config import ( "testing" - "github.com/rook/rook/pkg/daemon/ceph/client" - "github.com/rook/rook/pkg/operator/ceph/version" "github.com/stretchr/testify/assert" ) @@ -31,12 +29,3 @@ func TestNewFlag(t *testing.T) { assert.Equal(t, NewFlag("c key", "c"), "--c-key=c") assert.Equal(t, NewFlag("quotes", "\"quoted\""), "--quotes=\"quoted\"") } - -func TestInsecureGlobalIDVersion(t *testing.T) { - c := &client.ClusterInfo{CephVersion: version.CephVersion{Major: 17, Minor: 2, Extra: 0}} - assert.True(t, canDisableInsecureGlobalID(c)) - c = &client.ClusterInfo{CephVersion: version.CephVersion{Major: 16, Minor: 2, Extra: 0}} - assert.False(t, canDisableInsecureGlobalID(c)) - c = &client.ClusterInfo{CephVersion: version.CephVersion{Major: 16, Minor: 2, Extra: 1}} - assert.True(t, canDisableInsecureGlobalID(c)) -} diff --git a/pkg/operator/ceph/controller/network.go b/pkg/operator/ceph/controller/network.go index e4892f487496..4c64d37ab03f 100644 --- a/pkg/operator/ceph/controller/network.go +++ b/pkg/operator/ceph/controller/network.go @@ -250,7 +250,7 @@ func discoverAddressRanges( `}, []string{}, rookImage, - clusterSpec.CephVersion.Image, + rookImage, clusterSpec.CephVersion.ImagePullPolicy, ) if err != nil { @@ -266,13 +266,13 @@ func discoverAddressRanges( } // use osd placement for net canaries b/c osd pods are present on both public and cluster nets - clusterSpec.Placement[cephv1.KeyOSD].ApplyToPodSpec(&job.Spec.Template.Spec) + cephv1.GetOSDPlacement(clusterSpec.Placement).ApplyToPodSpec(&job.Spec.Template.Spec) // set up net status vol from downward api, plus init container to wait for net status to be available netStatusVol, netStatusMount := networkStatusVolumeAndMount() job.Spec.Template.Spec.Volumes = append(job.Spec.Template.Spec.Volumes, netStatusVol) job.Spec.Template.Spec.Containers[0].VolumeMounts = append(job.Spec.Template.Spec.Containers[0].VolumeMounts, netStatusMount) - job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, containerWaitForNetworkStatus(clusterSpec)) + job.Spec.Template.Spec.InitContainers = append(job.Spec.Template.Spec.InitContainers, containerWaitForNetworkStatus(clusterSpec, rookImage)) stdout, stderr, retcode, err := networkCanary.Run(ctx, detectNetworkCIDRTimeout) if err != nil { @@ -353,7 +353,7 @@ func networkStatusVolumeAndMount() (corev1.Volume, corev1.VolumeMount) { // to wait until the network status annotation is present to output info before running so that it // reports all the info. separate step of waiting for network status annotation into an init // container so that it can be more easily debugged (output not present in cmd reporter's result) -func containerWaitForNetworkStatus(clusterSpec *cephv1.ClusterSpec) corev1.Container { +func containerWaitForNetworkStatus(clusterSpec *cephv1.ClusterSpec, rookImage string) corev1.Container { _, mount := networkStatusVolumeAndMount() return corev1.Container{ Name: "wait-for-network-status-annotation", @@ -369,7 +369,7 @@ func containerWaitForNetworkStatus(clusterSpec *cephv1.ClusterSpec) corev1.Conta echo "" # newline `, }, - Image: clusterSpec.CephVersion.Image, + Image: rookImage, ImagePullPolicy: clusterSpec.CephVersion.ImagePullPolicy, VolumeMounts: []corev1.VolumeMount{ mount, diff --git a/pkg/operator/ceph/controller/network_test.go b/pkg/operator/ceph/controller/network_test.go index d8f289b4c298..8cc5beab1eb3 100644 --- a/pkg/operator/ceph/controller/network_test.go +++ b/pkg/operator/ceph/controller/network_test.go @@ -287,6 +287,74 @@ func Test_discoverAddressRanges(t *testing.T) { assert.Equal(t, []string{"10.144.0.0/16", "fe05::/96"}, ranges) }) }) + + t.Run("cluster config with all placement", func(t *testing.T) { + tolerations := []v1.Toleration{ + { + Key: "testkey", + Operator: v1.TolerationOpExists, + }, + } + clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec( + cephv1.NetworkSpec{ + Provider: cephv1.NetworkProviderMultus, + Selectors: map[cephv1.CephNetworkType]string{ + cephv1.CephNetworkPublic: "macvlan-public", + cephv1.CephNetworkCluster: "macvlan-cluster", + }, + AddressRanges: nil, // should auto-detect + }) + clusterSpec.Placement = map[cephv1.KeyType]cephv1.Placement{ + cephv1.KeyAll: {Tolerations: tolerations}, + } + + cmdReporter := new(mockCmdReporter) + cmdReporter.On("Job") + goodOutput := netStatus("public", "2000::1") + + "\n" + separator() + "\n" + + ipAddrOutput("public", "2000::1/112") + cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil) + newCmdReporter = mockNewCmdReporter(cmdReporter, nil) + + ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic) + assert.NoError(t, err) + assert.Equal(t, []string{"2000::/112"}, ranges) + assert.Equal(t, tolerations, cmdReporter.Job().Spec.Template.Spec.Tolerations) + }) + + t.Run("cluster config with osd placement", func(t *testing.T) { + tolerations := []v1.Toleration{ + { + Key: "testkey", + Operator: v1.TolerationOpExists, + }, + } + clusterdCtx, clusterSpec, clusterInfo := newTestConfigsWithNetworkSpec( + cephv1.NetworkSpec{ + Provider: cephv1.NetworkProviderMultus, + Selectors: map[cephv1.CephNetworkType]string{ + cephv1.CephNetworkPublic: "macvlan-public", + cephv1.CephNetworkCluster: "macvlan-cluster", + }, + AddressRanges: nil, // should auto-detect + }) + clusterSpec.Placement = map[cephv1.KeyType]cephv1.Placement{ + cephv1.KeyOSD: {Tolerations: tolerations}, + } + + cmdReporter := new(mockCmdReporter) + cmdReporter.On("Job") + goodOutput := netStatus("public", "2000::1") + + "\n" + separator() + "\n" + + ipAddrOutput("public", "2000::1/112") + cmdReporter.On("Run", mock.Anything, mock.Anything).Return(goodOutput, "", 0, nil) + newCmdReporter = mockNewCmdReporter(cmdReporter, nil) + + ranges, err := discoverAddressRanges(context.Background(), "rook/ceph:master", clusterdCtx, clusterSpec, clusterInfo, cephv1.CephNetworkPublic) + assert.NoError(t, err) + assert.Equal(t, []string{"2000::/112"}, ranges) + assert.Equal(t, tolerations, cmdReporter.Job().Spec.Template.Spec.Tolerations) + }) } func mockDiscoverAddressRangesFunc(publicReturn []string, clusterReturn []string, panicIfDiscoverPublic, panicIfDiscoverCluster bool, shouldErr error) func(ctx context.Context, rookImage string, clusterdContext *clusterd.Context, clusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo, discoverPublic bool, discoverCluster bool) (publicRanges []cephv1.CIDR, clusterRanges []cephv1.CIDR, err error) { diff --git a/pkg/operator/ceph/controller/spec.go b/pkg/operator/ceph/controller/spec.go index e9598ab8ba22..32caab2b4b71 100644 --- a/pkg/operator/ceph/controller/spec.go +++ b/pkg/operator/ceph/controller/spec.go @@ -18,6 +18,7 @@ limitations under the License. package controller import ( + "encoding/binary" "fmt" "os" "path" @@ -39,6 +40,7 @@ import ( kerrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" ) const ( @@ -388,8 +390,8 @@ func AdminFlags(cluster *client.ClusterInfo) []string { func NetworkBindingFlags(cluster *client.ClusterInfo, spec *cephv1.ClusterSpec) []string { var args []string - // As of Pacific, Ceph supports dual-stack, so setting IPv6 family without disabling IPv4 binding actually enables dual-stack - // This is likely not user's intent, so on Pacific let's make sure to disable IPv4 when IPv6 is selected + // Ceph supports dual-stack, so setting IPv6 family without disabling IPv4 binding actually enables dual-stack + // This is likely not user's intent, so let's make sure to disable IPv4 when IPv6 is selected if !spec.Network.DualStack { switch spec.Network.IPFamily { case cephv1.IPv4: @@ -898,3 +900,37 @@ func GetContainerImagePullPolicy(containerImagePullPolicy v1.PullPolicy) v1.Pull return containerImagePullPolicy } + +// GenerateLivenessProbeTcpPort generates a liveness probe that makes sure a daemon has +// TCP a socket binded to specific port, and may create new connection. +func GenerateLivenessProbeTcpPort(port, failureThreshold int32) *v1.Probe { + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + TCPSocket: &v1.TCPSocketAction{ + Port: intstr.IntOrString{IntVal: port}, + }, + }, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + TimeoutSeconds: livenessProbeTimeoutSeconds, + FailureThreshold: failureThreshold, + } +} + +// GenerateLivenessProbeViaRpcinfo creates a liveness probe using 'rpcinfo' shell +// command which checks that the local NFS daemon has TCP a socket binded to +// specific port, and it has valid reply to NULL RPC request. +func GenerateLivenessProbeViaRpcinfo(port uint16, failureThreshold int32) *v1.Probe { + bb := make([]byte, 2) + binary.BigEndian.PutUint16(bb, port) // port-num in network-order + servAddr := fmt.Sprintf("127.0.0.1.%d.%d", bb[0], bb[1]) + return &v1.Probe{ + ProbeHandler: v1.ProbeHandler{ + Exec: &v1.ExecAction{ + Command: []string{"rpcinfo", "-a", servAddr, "-T", "tcp", "nfs", "4"}, + }, + }, + InitialDelaySeconds: livenessProbeInitialDelaySeconds, + TimeoutSeconds: livenessProbeTimeoutSeconds, + FailureThreshold: failureThreshold, + } +} diff --git a/pkg/operator/ceph/controller/spec_test.go b/pkg/operator/ceph/controller/spec_test.go index 9acdad5d1a60..1c61e3f9cc28 100644 --- a/pkg/operator/ceph/controller/spec_test.go +++ b/pkg/operator/ceph/controller/spec_test.go @@ -229,9 +229,9 @@ func TestNetworkBindingFlags(t *testing.T) { args args want []string }{ - {"pacific-ipv4", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv4}}}, []string{ipv4FlagTrue, ipv6FlagFalse}}, - {"pacific-ipv6", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6}}}, []string{ipv4FlagFalse, ipv6FlagTrue}}, - {"pacific-dualstack-supported", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Pacific}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6, DualStack: true}}}, []string{ipv4FlagTrue, ipv6FlagTrue}}, + {"ipv4", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Reef}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv4}}}, []string{ipv4FlagTrue, ipv6FlagFalse}}, + {"ipv6", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Reef}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6}}}, []string{ipv4FlagFalse, ipv6FlagTrue}}, + {"dualstack-supported", args{cluster: &cephclient.ClusterInfo{CephVersion: version.Reef}, spec: &cephv1.ClusterSpec{Network: cephv1.NetworkSpec{IPFamily: cephv1.IPv6, DualStack: true}}}, []string{ipv4FlagTrue, ipv6FlagTrue}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { diff --git a/pkg/operator/ceph/csi/csi.go b/pkg/operator/ceph/csi/csi.go index c85364739d52..49d79423b4e3 100644 --- a/pkg/operator/ceph/csi/csi.go +++ b/pkg/operator/ceph/csi/csi.go @@ -17,7 +17,6 @@ limitations under the License. package csi import ( - "fmt" "strconv" "strings" "time" @@ -81,11 +80,6 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ALLOW_UNSUPPORTED_VERSION'") } - if EnableCSIGRPCMetrics, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "ROOK_CSI_ENABLE_GRPC_METRICS", "false")); err != nil { - return errors.Wrap(err, "unable to parse value for 'ROOK_CSI_ENABLE_GRPC_METRICS'") - } - CSIParam.EnableCSIGRPCMetrics = fmt.Sprintf("%t", EnableCSIGRPCMetrics) - if CSIParam.EnableCSIHostNetwork, err = strconv.ParseBool(k8sutil.GetValue(r.opConfig.Parameters, "CSI_ENABLE_HOST_NETWORK", "true")); err != nil { return errors.Wrap(err, "failed to parse value for 'CSI_ENABLE_HOST_NETWORK'") } @@ -115,20 +109,11 @@ func (r *ReconcileCSI) setParams(ver *version.Info) error { } CSIParam.GRPCTimeout = time.Duration(timeoutSeconds) * time.Second - // parse GRPC and Liveness ports - CSIParam.CephFSGRPCMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_GRPC_METRICS_PORT", DefaultCephFSGRPCMerticsPort) - if err != nil { - return errors.Wrap(err, "error getting CSI CephFS GRPC metrics port.") - } + // parse Liveness port CSIParam.CephFSLivenessMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_CEPHFS_LIVENESS_METRICS_PORT", DefaultCephFSLivenessMerticsPort) if err != nil { return errors.Wrap(err, "error getting CSI CephFS liveness metrics port.") } - - CSIParam.RBDGRPCMetricsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSI_RBD_GRPC_METRICS_PORT", DefaultRBDGRPCMerticsPort) - if err != nil { - return errors.Wrap(err, "error getting CSI RBD GRPC metrics port.") - } CSIParam.CSIAddonsPort, err = getPortFromConfig(r.opConfig.Parameters, "CSIADDONS_PORT", DefaultCSIAddonsPort) if err != nil { return errors.Wrap(err, "failed to get CSI Addons port") diff --git a/pkg/operator/ceph/csi/spec.go b/pkg/operator/ceph/csi/spec.go index 94743c03767c..025933dfb5d1 100644 --- a/pkg/operator/ceph/csi/spec.go +++ b/pkg/operator/ceph/csi/spec.go @@ -46,7 +46,6 @@ type Param struct { SnapshotterImage string ResizerImage string DriverNamePrefix string - EnableCSIGRPCMetrics string KubeletDirPath string ForceCephFSKernelClient string CephFSKernelMountOptions string @@ -83,9 +82,7 @@ type Param struct { NFSAttachRequired bool LogLevel uint8 SidecarLogLevel uint8 - CephFSGRPCMetricsPort uint16 CephFSLivenessMetricsPort uint16 - RBDGRPCMetricsPort uint16 CSIAddonsPort uint16 RBDLivenessMetricsPort uint16 ProvisionerReplicas int32 @@ -115,7 +112,6 @@ var ( EnableRBD = false EnableCephFS = false EnableNFS = false - EnableCSIGRPCMetrics = false AllowUnsupported = false CustomCSICephConfigExists = false @@ -321,7 +317,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI } // Create service if either liveness or GRPC metrics are enabled. - if CSIParam.EnableLiveness || EnableCSIGRPCMetrics { + if CSIParam.EnableLiveness { rbdService, err = templateToService("rbd-service", RBDPluginServiceTemplatePath, tp) if err != nil { return errors.Wrap(err, "failed to load rbd plugin service template") @@ -348,7 +344,7 @@ func (r *ReconcileCSI) startDrivers(ver *version.Info, ownerInfo *k8sutil.OwnerI return errors.Wrap(err, "failed to load rbd provisioner deployment template") } // Create service if either liveness or GRPC metrics are enabled. - if CSIParam.EnableLiveness || EnableCSIGRPCMetrics { + if CSIParam.EnableLiveness { cephfsService, err = templateToService("cephfs-service", CephFSPluginServiceTemplatePath, tp) if err != nil { return errors.Wrap(err, "failed to load cephfs plugin service template") diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml index f2ba703dfb81..14374971a3bf 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin-provisioner-dep.yaml @@ -112,10 +112,7 @@ spec: - "--controllerserver=true" - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport={{ .CephFSGRPCMetricsPort }}" - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - - "--metricspath=/metrics" - - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" {{ if .CSIEnableMetadata }} - "--setmetadata={{ .CSIEnableMetadata }}" {{ end }} diff --git a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml index 47fd21a021aa..1f2b306e4a2a 100644 --- a/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml +++ b/pkg/operator/ceph/csi/template/cephfs/csi-cephfsplugin.yaml @@ -71,10 +71,7 @@ spec: - "--nodeserver=true" - "--drivername={{ .DriverNamePrefix }}cephfs.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport={{ .CephFSGRPCMetricsPort }}" - "--forcecephkernelclient={{ .ForceCephFSKernelClient }}" - - "--metricspath=/metrics" - - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" {{ if .CephFSKernelMountOptions }} - "--kernelmountoptions={{ .CephFSKernelMountOptions }}" {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml index 872a27407fdd..99a710d05be0 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin-provisioner-dep.yaml @@ -181,9 +181,6 @@ spec: - "--controllerserver=true" - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport={{ .RBDGRPCMetricsPort }}" - - "--metricspath=/metrics" - - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" {{ if .EnableCSIAddonsSideCar }} - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" {{ end }} diff --git a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml index 8c023b2db10b..95153f4023a0 100644 --- a/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml +++ b/pkg/operator/ceph/csi/template/rbd/csi-rbdplugin.yaml @@ -72,9 +72,6 @@ spec: - "--nodeserver=true" - "--drivername={{ .DriverNamePrefix }}rbd.csi.ceph.com" - "--pidlimit=-1" - - "--metricsport={{ .RBDGRPCMetricsPort }}" - - "--metricspath=/metrics" - - "--enablegrpcmetrics={{ .EnableCSIGRPCMetrics }}" - "--stagingpath={{ .KubeletDirPath }}/plugins/kubernetes.io/csi/" {{ if .EnableCSIAddonsSideCar }} - "--csi-addons-endpoint=$(CSIADDONS_ENDPOINT)" diff --git a/pkg/operator/ceph/file/controller_test.go b/pkg/operator/ceph/file/controller_test.go index 1eb672000b3b..cf7f67d82816 100644 --- a/pkg/operator/ceph/file/controller_test.go +++ b/pkg/operator/ceph/file/controller_test.go @@ -159,7 +159,7 @@ func TestCephFilesystemController(t *testing.T) { os.Setenv("ROOK_LOG_LEVEL", "DEBUG") currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { - return &version.Pacific, &version.Pacific, nil + return &version.Reef, &version.Reef, nil } // A Pool resource with metadata and spec. diff --git a/pkg/operator/ceph/file/filesystem_test.go b/pkg/operator/ceph/file/filesystem_test.go index 68990550a332..30d69f5b8c45 100644 --- a/pkg/operator/ceph/file/filesystem_test.go +++ b/pkg/operator/ceph/file/filesystem_test.go @@ -377,7 +377,7 @@ func TestCreateFilesystem(t *testing.T) { }) t.Run("multi filesystem creation should succeed", func(t *testing.T) { - clusterInfo.CephVersion = version.Pacific + clusterInfo.CephVersion = version.Quincy err := createFilesystem(context, clusterInfo, fs, &cephv1.ClusterSpec{}, ownerInfo, "/var/lib/rook/") assert.NoError(t, err) }) @@ -399,7 +399,7 @@ func TestUpgradeFilesystem(t *testing.T) { ConfigDir: configDir, Clientset: clientset} fs := fsTest(fsName) - clusterInfo := &cephclient.ClusterInfo{FSID: "myfsid", CephVersion: version.Pacific, Context: ctx} + clusterInfo := &cephclient.ClusterInfo{FSID: "myfsid", CephVersion: version.Quincy, Context: ctx} // start a basic cluster for upgrade ownerInfo := cephclient.NewMinimumOwnerInfoWithOwnerRef() @@ -485,7 +485,7 @@ func TestUpgradeFilesystem(t *testing.T) { versionStr, _ := json.Marshal( map[string]map[string]int{ "mds": { - "ceph version 16.2.0-0-g2f728b9 (2f728b952cf293dd7f809ad8a0f5b5d040c43010) pacific (stable)": 2, + "ceph version 17.2.0-0-g2f728b9 (2f728b952cf293dd7f809ad8a0f5b5d040c43010) quincy (stable)": 2, }, }) return string(versionStr), nil @@ -494,7 +494,7 @@ func TestUpgradeFilesystem(t *testing.T) { return "", nil } // do upgrade - clusterInfo.CephVersion = version.Quincy + clusterInfo.CephVersion = version.Reef context = &clusterd.Context{ Executor: executor, ConfigDir: configDir, diff --git a/pkg/operator/ceph/file/mds/mds.go b/pkg/operator/ceph/file/mds/mds.go index d1aef87af61a..ff14e7836cc5 100644 --- a/pkg/operator/ceph/file/mds/mds.go +++ b/pkg/operator/ceph/file/mds/mds.go @@ -247,8 +247,7 @@ func (c *Cluster) upgradeMDS() error { return errors.Wrap(err, "failed to setting allow_standby_replay to false") } - // In Pacific, standby-replay daemons are stopped automatically. Older versions of Ceph require - // us to stop these daemons manually. + // Standby-replay daemons are stopped automatically. if err := cephclient.FailAllStandbyReplayMDS(c.context, c.clusterInfo, c.fs.Name); err != nil { return errors.Wrap(err, "failed to fail mds agent in up:standby-replay state") } diff --git a/pkg/operator/ceph/file/mirror/controller_test.go b/pkg/operator/ceph/file/mirror/controller_test.go index 393580c73327..1edcd6253ea1 100644 --- a/pkg/operator/ceph/file/mirror/controller_test.go +++ b/pkg/operator/ceph/file/mirror/controller_test.go @@ -228,7 +228,7 @@ func TestCephFilesystemMirrorController(t *testing.T) { } currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { - return &version.Pacific, &version.Quincy, nil + return &version.Quincy, &version.Reef, nil } res, err := r.Reconcile(ctx, req) @@ -236,7 +236,7 @@ func TestCephFilesystemMirrorController(t *testing.T) { assert.True(t, res.Requeue) }) - t.Run("success - running pacific", func(t *testing.T) { + t.Run("success create reef", func(t *testing.T) { r.context.Executor = &exectest.MockExecutor{ MockExecuteCommandWithOutput: func(command string, args ...string) (string, error) { if args[0] == "status" { @@ -250,7 +250,7 @@ func TestCephFilesystemMirrorController(t *testing.T) { } currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*version.CephVersion, *version.CephVersion, error) { - return &version.Pacific, &version.Pacific, nil + return &version.Reef, &version.Reef, nil } res, err := r.Reconcile(ctx, req) diff --git a/pkg/operator/ceph/nfs/controller.go b/pkg/operator/ceph/nfs/controller.go index 1d91cab7dad0..f625ac26e08c 100644 --- a/pkg/operator/ceph/nfs/controller.go +++ b/pkg/operator/ceph/nfs/controller.go @@ -31,7 +31,6 @@ import ( "github.com/rook/rook/pkg/operator/ceph/config" opcontroller "github.com/rook/rook/pkg/operator/ceph/controller" "github.com/rook/rook/pkg/operator/ceph/reporting" - "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" appsv1 "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -52,9 +51,6 @@ const ( controllerName = "ceph-nfs-controller" ) -// Version of Ceph where NFS default pool name changes to ".nfs" -var cephNFSChangeVersion = version.CephVersion{Major: 16, Minor: 2, Extra: 7} - var logger = capnslog.NewPackageLogger("github.com/rook/rook", controllerName) // List of object resources to watch by the controller @@ -269,13 +265,7 @@ func (r *ReconcileCephNFS) reconcile(request reconcile.Request) (reconcile.Resul } r.clusterInfo.CephVersion = *runningCephVersion - // Pacific before 16.2.7: No customization, default pool name is nfs-ganesha - // Pacific after 16.2.7: No customization, default pool name is .nfs - if r.clusterInfo.CephVersion.IsAtLeast(cephNFSChangeVersion) { - cephNFS.Spec.RADOS.Pool = postNFSChangeDefaultPoolName - } else { - cephNFS.Spec.RADOS.Pool = preNFSChangeDefaultPoolName - } + cephNFS.Spec.RADOS.Pool = nfsDefaultPoolName cephNFS.Spec.RADOS.Namespace = cephNFS.Name // validate the store settings diff --git a/pkg/operator/ceph/nfs/controller_test.go b/pkg/operator/ceph/nfs/controller_test.go index 884470391284..f03292ec063b 100644 --- a/pkg/operator/ceph/nfs/controller_test.go +++ b/pkg/operator/ceph/nfs/controller_test.go @@ -507,7 +507,7 @@ func TestGetGaneshaConfigObject(t *testing.T) { expectedName := "conf-nfs.my-nfs" res := getGaneshaConfigObject(cephNFS) - logger.Infof("Config Object for Pacific is %s", res) + logger.Infof("Config Object is %s", res) assert.Equal(t, expectedName, res) } diff --git a/pkg/operator/ceph/nfs/nfs.go b/pkg/operator/ceph/nfs/nfs.go index 7ac3b05588ab..d4ae2c76a3d9 100644 --- a/pkg/operator/ceph/nfs/nfs.go +++ b/pkg/operator/ceph/nfs/nfs.go @@ -37,9 +37,7 @@ import ( const ( // Default RADOS pool name after the NFS changes in Ceph - postNFSChangeDefaultPoolName = ".nfs" - // Default RADOS pool name before the NFS changes in Ceph - preNFSChangeDefaultPoolName = "nfs-ganesha" + nfsDefaultPoolName = ".nfs" // CephNFSNameLabelKey is the label key that contains the name of the CephNFS resource CephNFSNameLabelKey = "ceph_nfs" diff --git a/pkg/operator/ceph/nfs/spec.go b/pkg/operator/ceph/nfs/spec.go index 8c857edf48ad..4c4bcbf45e8d 100644 --- a/pkg/operator/ceph/nfs/spec.go +++ b/pkg/operator/ceph/nfs/spec.go @@ -22,8 +22,10 @@ import ( "github.com/pkg/errors" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" + cephconfig "github.com/rook/rook/pkg/operator/ceph/config" "github.com/rook/rook/pkg/operator/ceph/config/keyring" "github.com/rook/rook/pkg/operator/ceph/controller" + "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" apps "k8s.io/api/apps/v1" v1 "k8s.io/api/core/v1" @@ -224,7 +226,7 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig logLevel = nfs.Spec.Server.LogLevel } - return v1.Container{ + container := v1.Container{ Name: "nfs-ganesha", Command: []string{ "ganesha.nfsd", @@ -246,7 +248,20 @@ func (r *ReconcileCephNFS) daemonContainer(nfs *cephv1.CephNFS, cfg daemonConfig Env: controller.DaemonEnvVars(r.cephClusterSpec), Resources: nfs.Spec.Server.Resources, SecurityContext: controller.PodSecurityContext(), + LivenessProbe: r.defaultGaneshaLivenessProbe(nfs), + } + return cephconfig.ConfigureLivenessProbe(container, nfs.Spec.Server.LivenessProbe) +} + +func (r *ReconcileCephNFS) defaultGaneshaLivenessProbe(nfs *cephv1.CephNFS) *v1.Probe { + failureThreshold := int32(10) + cephVersionWithRpcinfo := version.CephVersion{Major: 18, Minor: 2, Extra: 1} + if r.clusterInfo.CephVersion.IsAtLeast(cephVersionWithRpcinfo) { + // liveness-probe using rpcinfo utility + return controller.GenerateLivenessProbeViaRpcinfo(nfsPort, failureThreshold) } + // liveness-probe using K8s builtin TCP-socket action + return controller.GenerateLivenessProbeTcpPort(nfsPort, failureThreshold) } func (r *ReconcileCephNFS) dbusContainer(nfs *cephv1.CephNFS) v1.Container { diff --git a/pkg/operator/ceph/nfs/spec_test.go b/pkg/operator/ceph/nfs/spec_test.go index 0e729a2bfc66..548faeb830fd 100644 --- a/pkg/operator/ceph/nfs/spec_test.go +++ b/pkg/operator/ceph/nfs/spec_test.go @@ -402,4 +402,55 @@ func TestDeploymentSpec(t *testing.T) { } assert.Contains(t, contNames, "sssd") }) + + t.Run("basic with default liveness-probe", func(t *testing.T) { + nfs := &cephv1.CephNFS{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-nfs", + Namespace: "rook-ceph-test-ns", + }, + Spec: cephv1.NFSGaneshaSpec{ + RADOS: cephv1.GaneshaRADOSSpec{ + Pool: "myfs-data0", + Namespace: "nfs-test-ns", + }, + Server: cephv1.GaneshaServerSpec{ + Active: 3, + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(500.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(1024.0, resource.BinarySI), + }, + Requests: v1.ResourceList{ + v1.ResourceCPU: *resource.NewQuantity(200.0, resource.BinarySI), + v1.ResourceMemory: *resource.NewQuantity(512.0, resource.BinarySI), + }, + }, + PriorityClassName: "my-priority-class", + LivenessProbe: &cephv1.ProbeSpec{ + Disabled: false, + }, + }, + }, + } + + r, cfg := newDeploymentSpecTest(t) + d, err := r.makeDeployment(nfs, cfg) + assert.NoError(t, err) + assert.NotEmpty(t, d.Spec.Template.Annotations) + + // Expects valid settings to default livness-probe + var ganeshaCont *v1.Container = nil + for i := range d.Spec.Template.Spec.Containers { + if d.Spec.Template.Spec.Containers[i].Name == "nfs-ganesha" { + ganeshaCont = &d.Spec.Template.Spec.Containers[i] + break + } + } + assert.NotNil(t, ganeshaCont) + assert.NotNil(t, ganeshaCont.LivenessProbe) + assert.Equal(t, ganeshaCont.LivenessProbe.InitialDelaySeconds, int32(10)) + assert.Equal(t, ganeshaCont.LivenessProbe.FailureThreshold, int32(10)) + assert.GreaterOrEqual(t, ganeshaCont.LivenessProbe.TimeoutSeconds, int32(5)) + }) } diff --git a/pkg/operator/ceph/object/controller_test.go b/pkg/operator/ceph/object/controller_test.go index d58e1a3df149..6c3fb93ffb03 100644 --- a/pkg/operator/ceph/object/controller_test.go +++ b/pkg/operator/ceph/object/controller_test.go @@ -444,7 +444,7 @@ func TestCephObjectStoreController(t *testing.T) { } currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*cephver.CephVersion, *cephver.CephVersion, error) { - return &cephver.Pacific, &cephver.Pacific, nil + return &cephver.Reef, &cephver.Reef, nil } t.Run("error - no ceph cluster", func(t *testing.T) { @@ -580,7 +580,7 @@ func TestCephObjectStoreController(t *testing.T) { assert.NotEmpty(t, objectStore.Status.Info["endpoint"], objectStore) assert.Equal(t, "http://rook-ceph-rgw-my-store.rook-ceph.svc:80", objectStore.Status.Info["endpoint"], objectStore) assert.True(t, calledCommitConfigChanges) - assert.Equal(t, 16, r.clusterInfo.CephVersion.Major) + assert.Equal(t, 18, r.clusterInfo.CephVersion.Major) }) } @@ -781,7 +781,7 @@ func TestCephObjectStoreControllerMultisite(t *testing.T) { } currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*cephver.CephVersion, *cephver.CephVersion, error) { - return &cephver.Pacific, &cephver.Pacific, nil + return &cephver.Reef, &cephver.Reef, nil } t.Run("create an object store", func(t *testing.T) { @@ -953,7 +953,7 @@ func TestCephObjectExternalStoreController(t *testing.T) { } currentAndDesiredCephVersion = func(ctx context.Context, rookImage string, namespace string, jobName string, ownerInfo *k8sutil.OwnerInfo, context *clusterd.Context, cephClusterSpec *cephv1.ClusterSpec, clusterInfo *client.ClusterInfo) (*cephver.CephVersion, *cephver.CephVersion, error) { - return &cephver.Pacific, &cephver.Pacific, nil + return &cephver.Reef, &cephver.Reef, nil } { diff --git a/pkg/operator/ceph/object/objectstore.go b/pkg/operator/ceph/object/objectstore.go index 5ca80e7783af..0b1f984fe410 100644 --- a/pkg/operator/ceph/object/objectstore.go +++ b/pkg/operator/ceph/object/objectstore.go @@ -70,9 +70,8 @@ var ( dataPoolName = "rgw.buckets.data" // An user with system privileges for dashboard service - DashboardUser = "dashboard-admin" - rgwPgNumRemovedQuincyVersion = cephver.CephVersion{Major: 17, Minor: 2, Extra: 2} - rgwPgNumRemovedPacificVersion = cephver.CephVersion{Major: 16, Minor: 2, Extra: 11} + DashboardUser = "dashboard-admin" + rgwPgNumRemovedQuincyVersion = cephver.CephVersion{Major: 17, Minor: 2, Extra: 2} ) type idType struct { @@ -389,55 +388,61 @@ func getZoneEndpoints(objContext *Context, serviceEndpoint string) ([]string, bo return zoneEndpointsList, isEndpointAlreadyExists, nil } +func createMultisiteConfigurations(objContext *Context, configType, configTypeArg string, args ...string) error { + args = append([]string{configType}, args...) + args = append(args, configTypeArg) + // get the multisite config before creating + output, getConfigErr := RunAdminCommandNoMultisite(objContext, true, configType, "get", configTypeArg) + if getConfigErr == nil { + return nil + } + + if kerrors.IsNotFound(getConfigErr) { + // the pod used to exec command (act as a proxy) is not found/ready yet + // caller can nicely handle error and not overflow logs with misleading error messages + return getConfigErr + } + + code, err := exec.ExtractExitCode(getConfigErr) + if err != nil { + return errorOrIsNotFound(getConfigErr, "'radosgw-admin %q get' failed with code %q, for reason %q, error: (%v)", configType, strconv.Itoa(code), output, string(kerrors.ReasonForError(err))) + } + // ENOENT means "No such file or directory" + if code != int(syscall.ENOENT) { + code := strconv.Itoa(code) + return errors.Wrapf(getConfigErr, "'radosgw-admin %q get' failed with code %q, for reason %q", configType, code, output) + } + + // create the object if it doesn't exist yet + output, err = RunAdminCommandNoMultisite(objContext, false, args...) + if err != nil { + return errorOrIsNotFound(err, "failed to create ceph %q %q, for reason %q", configType, configTypeArg, output) + } + logger.Debugf("created %q %q", configType, configTypeArg) + + return nil +} + func createMultisite(objContext *Context, endpointArg string) error { logger.Debugf("creating realm, zone group, zone for object-store %v", objContext.Name) realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) zoneGroupArg := fmt.Sprintf("--rgw-zonegroup=%s", objContext.ZoneGroup) + zoneArg := fmt.Sprintf("--rgw-zone=%s", objContext.Zone) - // create the realm if it doesn't exist yet - output, err := RunAdminCommandNoMultisite(objContext, true, "realm", "get", realmArg) + err := createMultisiteConfigurations(objContext, "realm", realmArg, "create") if err != nil { - // ENOENT means "No such file or directory" - if code, err := exec.ExtractExitCode(err); err == nil && code == int(syscall.ENOENT) { - output, err = RunAdminCommandNoMultisite(objContext, false, "realm", "create", realmArg) - if err != nil { - return errorOrIsNotFound(err, "failed to create ceph realm %q, for reason %q", objContext.ZoneGroup, output) - } - logger.Debugf("created realm %q", objContext.Realm) - } else { - return errorOrIsNotFound(err, "'radosgw-admin realm get' failed with code %d, for reason %q. %v", strconv.Itoa(code), output, string(kerrors.ReasonForError(err))) - } + return err } - // create the zonegroup if it doesn't exist yet - output, err = RunAdminCommandNoMultisite(objContext, true, "zonegroup", "get", realmArg, zoneGroupArg) + err = createMultisiteConfigurations(objContext, "zonegroup", zoneGroupArg, "create", "--master", realmArg, endpointArg) if err != nil { - // ENOENT means "No such file or directory" - if code, err := exec.ExtractExitCode(err); err == nil && code == int(syscall.ENOENT) { - output, err = RunAdminCommandNoMultisite(objContext, false, "zonegroup", "create", "--master", realmArg, zoneGroupArg, endpointArg) - if err != nil { - return errorOrIsNotFound(err, "failed to create ceph zone group %q, for reason %q", objContext.ZoneGroup, output) - } - logger.Debugf("created zone group %q", objContext.ZoneGroup) - } else { - return errorOrIsNotFound(err, "'radosgw-admin zonegroup get' failed with code %d, for reason %q", strconv.Itoa(code), output) - } + return err } - // create the zone if it doesn't exist yet - output, err = runAdminCommand(objContext, true, "zone", "get") + err = createMultisiteConfigurations(objContext, "zone", zoneArg, "create", "--master", endpointArg, realmArg, zoneGroupArg) if err != nil { - // ENOENT means "No such file or directory" - if code, err := exec.ExtractExitCode(err); err == nil && code == int(syscall.ENOENT) { - output, err = runAdminCommand(objContext, false, "zone", "create", "--master", endpointArg) - if err != nil { - return errorOrIsNotFound(err, "failed to create ceph zone %q, for reason %q", objContext.Zone, output) - } - logger.Debugf("created zone %q", objContext.Zone) - } else { - return errorOrIsNotFound(err, "'radosgw-admin zone get' failed with code %d, for reason %q", strconv.Itoa(code), output) - } + return err } if err := commitConfigChanges(objContext); err != nil { @@ -769,14 +774,7 @@ func CreatePools(context *Context, clusterSpec *cephv1.ClusterSpec, metadataPool // Check if this is a recent release of ceph where the legacy rgw_rados_pool_pg_num_min // is no longer available. func rgwRadosPGNumIsNew(cephVer cephver.CephVersion) bool { - if cephVer.IsPacific() && cephVer.IsAtLeast(rgwPgNumRemovedPacificVersion) { - return true - } - if cephVer.IsAtLeast(rgwPgNumRemovedQuincyVersion) { - return true - } - // the legacy setting should still be checked - return false + return cephVer.IsAtLeast(rgwPgNumRemovedQuincyVersion) } // configurePoolsConcurrently checks if operator pod resources are set or not diff --git a/pkg/operator/ceph/object/objectstore_test.go b/pkg/operator/ceph/object/objectstore_test.go index 39368643190b..62cd8e9f7eea 100644 --- a/pkg/operator/ceph/object/objectstore_test.go +++ b/pkg/operator/ceph/object/objectstore_test.go @@ -29,12 +29,14 @@ import ( "github.com/rook/rook/pkg/daemon/ceph/client" cephver "github.com/rook/rook/pkg/operator/ceph/version" "github.com/rook/rook/pkg/operator/k8sutil" + "github.com/rook/rook/pkg/util/exec" exectest "github.com/rook/rook/pkg/util/exec/test" "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" k8sfake "k8s.io/client-go/kubernetes/fake" + kexec "k8s.io/utils/exec" ) const ( @@ -238,9 +240,6 @@ func TestGetObjectBucketProvisioner(t *testing.T) { } func TestRGWPGNumVersion(t *testing.T) { - assert.False(t, rgwRadosPGNumIsNew(cephver.CephVersion{Major: 15, Minor: 2, Extra: 9})) - assert.False(t, rgwRadosPGNumIsNew(cephver.CephVersion{Major: 16, Minor: 2, Extra: 10})) - assert.True(t, rgwRadosPGNumIsNew(cephver.CephVersion{Major: 16, Minor: 2, Extra: 11})) assert.False(t, rgwRadosPGNumIsNew(cephver.CephVersion{Major: 17, Minor: 2, Extra: 1})) assert.True(t, rgwRadosPGNumIsNew(cephver.CephVersion{Major: 17, Minor: 2, Extra: 2})) assert.True(t, rgwRadosPGNumIsNew(cephver.CephVersion{Major: 18, Minor: 0, Extra: 0})) @@ -669,6 +668,77 @@ func Test_createMultisite(t *testing.T) { } } +func getExecutor() []exec.Executor { + executor := []exec.Executor{&exectest.MockExecutor{ + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + if args[0] == "realm" { + return `{ + "id": "237e6250-5f7d-4b85-9359-8cb2b1848507", + "name": "realm-a", + "current_period": "df665ecb-1762-47a9-9c66-f938d251c02a", + "epoch": 2 +}`, nil + } + return "", nil + }, + }, + &exectest.MockExecutor{ + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + if args[0] == "realm" { + return `{}`, errors.Errorf("Error from server (NotFound): pods not found") + } + return "", nil + }, + }, + &exectest.MockExecutor{ + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + if args[0] == "realm" { + return `{}`, &kexec.CodeExitError{Err: errors.New("some error"), Code: 4} + } + return "", nil + }, + }, + &exectest.MockExecutor{ + MockExecuteCommandWithTimeout: func(timeout time.Duration, command string, args ...string) (string, error) { + if args[0] == "realm" { + return `{}`, &kexec.CodeExitError{Err: errors.New("some other error"), Code: 2} + } + return "", nil + }, + }, + } + return executor +} + +func getreturnErrString() []string { + returnErr := []string{ + "", + "'radosgw-admin [\"realm\" \"-1\" \"{}. \" \"\"] get' failed with code %!q(MISSING), for reason %!q(MISSING), error: (%!v(MISSING)): Error from server (NotFound): pods not found", + "'radosgw-admin \"realm\" get' failed with code \"4\", for reason \"{}. \": some error", + "failed to create ceph [\"realm\" \"--rgw-realm=\" \"{}. \"] %!q(MISSING), for reason %!q(MISSING): some other error", + } + return returnErr +} + +func Test_createMultisiteConfigurations(t *testing.T) { + executor := getExecutor() + returnErrString := getreturnErrString() + for i := 0; i < 4; i++ { + ctx := &clusterd.Context{ + Executor: executor[i], + } + objContext := NewContext(ctx, &client.ClusterInfo{Namespace: "my-cluster"}, "my-store") + realmArg := fmt.Sprintf("--rgw-realm=%s", objContext.Realm) + + err := createMultisiteConfigurations(objContext, "realm", realmArg, "create") + if i == 0 { + assert.NoError(t, err) + } else { + assert.Contains(t, err.Error(), returnErrString[i]) + } + } +} + func TestGetRealmKeySecret(t *testing.T) { ns := "my-ns" realmName := "my-realm" diff --git a/pkg/operator/ceph/object/topic/provisioner.go b/pkg/operator/ceph/object/topic/provisioner.go index 8790eeb89ede..eb41a2b7cb53 100644 --- a/pkg/operator/ceph/object/topic/provisioner.go +++ b/pkg/operator/ceph/object/topic/provisioner.go @@ -43,7 +43,6 @@ import ( "github.com/rook/rook/pkg/clusterd" cephclient "github.com/rook/rook/pkg/daemon/ceph/client" "github.com/rook/rook/pkg/operator/ceph/object" - "github.com/rook/rook/pkg/operator/ceph/version" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -144,7 +143,7 @@ func createSNSClient(p provisioner, objectStoreName types.NamespacedName) (*sns. return snsClient, nil } -func createTopicAttributes(topic *cephv1.CephBucketTopic, cephVersion version.CephVersion) map[string]*string { +func createTopicAttributes(topic *cephv1.CephBucketTopic) map[string]*string { attr := make(map[string]*string) nsName := types.NamespacedName{Name: topic.Name, Namespace: topic.Namespace} @@ -166,12 +165,8 @@ func createTopicAttributes(topic *cephv1.CephBucketTopic, cephVersion version.Ce attr["push-endpoint"] = &topic.Spec.Endpoint.HTTP.URI verifySSL = strconv.FormatBool(!topic.Spec.Endpoint.HTTP.DisableVerifySSL) attr["verify-ssl"] = &verifySSL - if cephVersion.IsAtLeast(version.Quincy) { - cloudEvents := strconv.FormatBool(topic.Spec.Endpoint.HTTP.SendCloudEvents) - attr["cloudevents"] = &cloudEvents - } else if topic.Spec.Endpoint.HTTP.SendCloudEvents { - logger.Warning("HTTP CloudEvents endpoint supported for Ceph Quincy (v17) or newer") - } + cloudEvents := strconv.FormatBool(topic.Spec.Endpoint.HTTP.SendCloudEvents) + attr["cloudevents"] = &cloudEvents } if topic.Spec.Endpoint.Kafka != nil { logger.Infof("creating CephBucketTopic %q with endpoint %q", nsName, topic.Spec.Endpoint.Kafka.URI) @@ -198,7 +193,7 @@ func createTopic(p provisioner, topic *cephv1.CephBucketTopic) (*string, error) } topicOutput, err := snsClient.CreateTopic(&sns.CreateTopicInput{ Name: &topic.Name, - Attributes: createTopicAttributes(topic, p.clusterInfo.CephVersion), + Attributes: createTopicAttributes(topic), }) if err != nil { diff --git a/pkg/operator/ceph/object/topic/provisioner_test.go b/pkg/operator/ceph/object/topic/provisioner_test.go index 79ee45ebc9b5..23ee17257b87 100644 --- a/pkg/operator/ceph/object/topic/provisioner_test.go +++ b/pkg/operator/ceph/object/topic/provisioner_test.go @@ -23,7 +23,6 @@ import ( "github.com/coreos/pkg/capnslog" cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" - "github.com/rook/rook/pkg/operator/ceph/version" "github.com/stretchr/testify/assert" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -63,15 +62,7 @@ func TestTopicAttributesCreation(t *testing.T) { }, }, } - assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic, version.Quincy)) - // make sure that non quincy version don't use the cloudevents flag - bucketTopic.Spec.Endpoint.HTTP.SendCloudEvents = true - delete(expectedAttrs, "cloudevents") - assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic, version.Pacific)) - // make sure that quincy version can use the cloudevents flag - expectedAttrs["cloudevents"] = &trueString - assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic, version.Quincy)) - + assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic)) }) t.Run("test AMQP attributes", func(t *testing.T) { uri := "amqp://my-rabbitmq-service:5672/vhost1" @@ -105,7 +96,7 @@ func TestTopicAttributesCreation(t *testing.T) { }, }, } - assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic, version.Quincy)) + assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic)) }) t.Run("test Kafka attributes", func(t *testing.T) { uri := "kafka://my-kafka-service:9092" @@ -138,6 +129,6 @@ func TestTopicAttributesCreation(t *testing.T) { }, }, } - assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic, version.Quincy)) + assert.Equal(t, expectedAttrs, createTopicAttributes(bucketTopic)) }) } diff --git a/pkg/operator/ceph/version/version.go b/pkg/operator/ceph/version/version.go index b6c87c096a3a..c27c20e203db 100644 --- a/pkg/operator/ceph/version/version.go +++ b/pkg/operator/ceph/version/version.go @@ -40,10 +40,8 @@ const ( var ( // Minimum supported version - Minimum = CephVersion{16, 2, 0, 0, ""} + Minimum = CephVersion{17, 2, 0, 0, ""} - // Pacific Ceph version - Pacific = CephVersion{16, 0, 0, 0, ""} // Quincy Ceph version Quincy = CephVersion{17, 0, 0, 0, ""} // Reef Ceph version @@ -52,7 +50,7 @@ var ( Squid = CephVersion{19, 0, 0, 0, ""} // supportedVersions are production-ready versions that rook supports - supportedVersions = []CephVersion{Pacific, Quincy, Reef} + supportedVersions = []CephVersion{Quincy, Reef} // unsupportedVersions are possibly Ceph pin-point release that introduced breaking changes and not recommended unsupportedVersions []CephVersion @@ -86,8 +84,6 @@ func (v *CephVersion) CephVersionFormatted() string { // ReleaseName is the name of the Ceph release func (v *CephVersion) ReleaseName() string { switch v.Major { - case Pacific.Major: - return "pacific" case Quincy.Major: return "quincy" case Reef.Major: @@ -169,11 +165,6 @@ func (v *CephVersion) isExactly(other CephVersion) bool { return v.Major == other.Major && v.Minor == other.Minor && v.Extra == other.Extra } -// IsPacific checks if the Ceph version is Pacific -func (v *CephVersion) IsPacific() bool { - return v.isRelease(Pacific) -} - // IsQuincy checks if the Ceph version is Quincy func (v *CephVersion) IsQuincy() bool { return v.isRelease(Quincy) diff --git a/pkg/operator/ceph/version/version_test.go b/pkg/operator/ceph/version/version_test.go index 31b2cfc25c7c..c1d47eba19d8 100644 --- a/pkg/operator/ceph/version/version_test.go +++ b/pkg/operator/ceph/version/version_test.go @@ -50,24 +50,24 @@ func extractVersionHelper(t *testing.T, text string, major, minor, extra, build func TestExtractVersion(t *testing.T) { // release build - v0c := "ceph version 16.2.6 (ae699615bac534ea496ee965ac6192cb7e0e07c1) pacific (stable)" + v0c := "ceph version 18.2.6 (ae699615bac534ea496ee965ac6192cb7e0e07c1) reef (stable)" v0d := ` root@7a97f5a78bc6:/# ceph --version -ceph version 16.2.6 (ae699615bac534ea496ee965ac6192cb7e0e07c1) pacific (stable) +ceph version 18.2.6 (ae699615bac534ea496ee965ac6192cb7e0e07c1) reef (stable) ` - extractVersionHelper(t, v0c, 16, 2, 6, 0, "ae699615bac534ea496ee965ac6192cb7e0e07c1") - extractVersionHelper(t, v0d, 16, 2, 6, 0, "ae699615bac534ea496ee965ac6192cb7e0e07c1") + extractVersionHelper(t, v0c, 18, 2, 6, 0, "ae699615bac534ea496ee965ac6192cb7e0e07c1") + extractVersionHelper(t, v0d, 18, 2, 6, 0, "ae699615bac534ea496ee965ac6192cb7e0e07c1") // development build - v1c := "ceph version 16.1.33-403-g7ba6bece41 (7ba6bece4187eda5d05a9b84211fe6ba8dd287bd) pacific (rc)" + v1c := "ceph version 18.1.33-403-g7ba6bece41 (7ba6bece4187eda5d05a9b84211fe6ba8dd287bd) reef (rc)" v1d := ` bin/ceph --version *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** -ceph version 16.1.33-403-g7ba6bece41 -(7ba6bece4187eda5d05a9b84211fe6ba8dd287bd) pacific (rc) +ceph version 18.1.33-403-g7ba6bece41 +(7ba6bece4187eda5d05a9b84211fe6ba8dd287bd) reef (rc) ` - extractVersionHelper(t, v1c, 16, 1, 33, 403, "7ba6bece4187eda5d05a9b84211fe6ba8dd287bd") - extractVersionHelper(t, v1d, 16, 1, 33, 403, "7ba6bece4187eda5d05a9b84211fe6ba8dd287bd") + extractVersionHelper(t, v1c, 18, 1, 33, 403, "7ba6bece4187eda5d05a9b84211fe6ba8dd287bd") + extractVersionHelper(t, v1d, 18, 1, 33, 403, "7ba6bece4187eda5d05a9b84211fe6ba8dd287bd") // build without git version info. it is possible to build the ceph tree // without a version number, but none of the container builds do this. @@ -75,7 +75,7 @@ ceph version 16.1.33-403-g7ba6bece41 // explicitly adding fine-grained versioning to avoid issues with // release granularity. adding the reverse name-to-version is easy // enough if this ever becomes a need. - v2c := "ceph version Development (no_version) pacific (rc)" + v2c := "ceph version Development (no_version) reef (rc)" v2d := ` bin/ceph --version *** DEVELOPER MODE: setting PATH, PYTHONPATH and LD_LIBRARY_PATH *** @@ -90,11 +90,11 @@ ceph version Development (no_version) quincy (rc) assert.Nil(t, v) // Test the round trip for serializing and deserializing the version - v3c := "ceph version 16.2.5-1 pacific" + v3c := "ceph version 18.2.5-1 reef" v, err = ExtractCephVersion(v3c) assert.NoError(t, err) assert.NotNil(t, v) - assert.Equal(t, "16.2.5-1 pacific", v.String()) + assert.Equal(t, "18.2.5-1 reef", v.String()) } func TestSupported(t *testing.T) { @@ -105,11 +105,9 @@ func TestSupported(t *testing.T) { } func TestIsRelease(t *testing.T) { - assert.True(t, Pacific.isRelease(Pacific)) assert.True(t, Quincy.isRelease(Quincy)) assert.True(t, Reef.isRelease(Reef)) - assert.False(t, Pacific.isRelease(Reef)) assert.False(t, Reef.isRelease(Quincy)) QuincyUpdate := Quincy @@ -119,7 +117,7 @@ func TestIsRelease(t *testing.T) { } func TestIsReleaseX(t *testing.T) { - assert.False(t, Pacific.IsQuincy()) + assert.False(t, Quincy.IsReef()) } func TestVersionAtLeast(t *testing.T) { @@ -136,7 +134,7 @@ func TestVersionAtLeast(t *testing.T) { func TestVersionAtLeastX(t *testing.T) { assert.True(t, Quincy.IsAtLeastQuincy()) - assert.False(t, Pacific.IsAtLeastQuincy()) + assert.False(t, Quincy.IsAtLeastReef()) } func TestIsIdentical(t *testing.T) { @@ -207,7 +205,6 @@ func TestCephVersion_Unsupported(t *testing.T) { fields fields want bool }{ - {"pacific", fields{Major: 16, Minor: 2, Extra: 1, Build: 0}, false}, {"quincy", fields{Major: 17, Minor: 2, Extra: 0, Build: 0}, false}, {"reef", fields{Major: 18, Minor: 2, Extra: 0, Build: 0}, false}, } diff --git a/pkg/operator/k8sutil/prometheus_test.go b/pkg/operator/k8sutil/prometheus_test.go index b2166138270d..6802896c597a 100644 --- a/pkg/operator/k8sutil/prometheus_test.go +++ b/pkg/operator/k8sutil/prometheus_test.go @@ -18,19 +18,21 @@ limitations under the License. package k8sutil import ( - "testing" - + monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" "github.com/stretchr/testify/assert" + "testing" ) func TestGetServiceMonitor(t *testing.T) { name := "rook-ceph-mgr" namespace := "rook-ceph" port := "http-metrics" + interval := monitoringv1.Duration("5s") servicemonitor := GetServiceMonitor(name, namespace, port) assert.Equal(t, name, servicemonitor.GetName()) assert.Equal(t, namespace, servicemonitor.GetNamespace()) assert.Equal(t, port, servicemonitor.Spec.Endpoints[0].Port) + assert.Equal(t, interval, servicemonitor.Spec.Endpoints[0].Interval) assert.NotNil(t, servicemonitor.GetLabels()) assert.NotNil(t, servicemonitor.Spec.NamespaceSelector.MatchNames) assert.NotNil(t, servicemonitor.Spec.Selector.MatchLabels) diff --git a/tests/framework/clients/object.go b/tests/framework/clients/object.go index 587d43cc4657..a5df6584702d 100644 --- a/tests/framework/clients/object.go +++ b/tests/framework/clients/object.go @@ -48,7 +48,7 @@ func (o *ObjectOperation) Create(namespace, storeName string, replicaCount int32 } // Starting an object store takes longer than the average operation, so add more retries - err := o.k8sh.WaitForLabeledPodsToRunWithRetries(fmt.Sprintf("rook_object_store=%s", storeName), namespace, 40) + err := o.k8sh.WaitForLabeledPodsToRunWithRetries(fmt.Sprintf("rook_object_store=%s", storeName), namespace, 80) if err != nil { return fmt.Errorf("rgw did not start via crd. %+v", err) } diff --git a/tests/framework/installer/ceph_installer.go b/tests/framework/installer/ceph_installer.go index aff970a21a94..14a795a387c8 100644 --- a/tests/framework/installer/ceph_installer.go +++ b/tests/framework/installer/ceph_installer.go @@ -42,16 +42,12 @@ import ( ) const ( - // test with the latest pacific build - pacificTestImage = "quay.io/ceph/ceph:v16" - // test with the latest quincy build + // test with the latest releases quincyTestImage = "quay.io/ceph/ceph:v17" - // test with the latest reef build - reefTestImage = "quay.io/ceph/ceph:v18" - // test with the current development version of Pacific - pacificDevelTestImage = "quay.io/ceph/daemon-base:latest-pacific-devel" - quincyDevelTestImage = "quay.io/ceph/daemon-base:latest-quincy-devel" - reefDevelTestImage = "quay.io/ceph/daemon-base:latest-reef-devel" + reefTestImage = "quay.io/ceph/ceph:v18" + // test with the current development versions + quincyDevelTestImage = "quay.io/ceph/daemon-base:latest-quincy-devel" + reefDevelTestImage = "quay.io/ceph/daemon-base:latest-reef-devel" // test with the latest Ceph main image mainTestImage = "quay.io/ceph/daemon-base:latest-main-devel" cephOperatorLabel = "app=rook-ceph-operator" @@ -71,8 +67,6 @@ mon compact on start = true ) var ( - PacificVersion = cephv1.CephVersionSpec{Image: pacificTestImage} - PacificDevelVersion = cephv1.CephVersionSpec{Image: pacificDevelTestImage} QuincyVersion = cephv1.CephVersionSpec{Image: quincyTestImage} QuincyDevelVersion = cephv1.CephVersionSpec{Image: quincyDevelTestImage} ReefVersion = cephv1.CephVersionSpec{Image: reefTestImage} @@ -99,8 +93,6 @@ func ReturnCephVersion() cephv1.CephVersionSpec { switch os.Getenv("CEPH_SUITE_VERSION") { case "main": return MainVersion - case "pacific-devel": - return PacificDevelVersion case "quincy-devel": return QuincyDevelVersion case "reef-devel": diff --git a/tests/framework/installer/ceph_manifests.go b/tests/framework/installer/ceph_manifests.go index 882b36875124..d0bc1e3134d6 100644 --- a/tests/framework/installer/ceph_manifests.go +++ b/tests/framework/installer/ceph_manifests.go @@ -71,7 +71,7 @@ func NewCephManifests(settings *TestCephSettings) CephManifests { switch settings.RookVersion { case LocalBuildTag: return &CephManifestsMaster{settings} - case Version1_11: + case Version1_12: return &CephManifestsPreviousVersion{settings, &CephManifestsMaster{settings}} } panic(fmt.Errorf("unrecognized ceph manifest version: %s", settings.RookVersion)) diff --git a/tests/framework/installer/ceph_manifests_previous.go b/tests/framework/installer/ceph_manifests_previous.go index 895a512fc129..ff41757d7a47 100644 --- a/tests/framework/installer/ceph_manifests_previous.go +++ b/tests/framework/installer/ceph_manifests_previous.go @@ -24,7 +24,7 @@ import ( const ( // The version from which the upgrade test will start - Version1_11 = "v1.11.7" + Version1_12 = "v1.12.7" ) // CephManifestsPreviousVersion wraps rook yaml definitions diff --git a/tests/framework/utils/k8s_helper.go b/tests/framework/utils/k8s_helper.go index c90c42de161c..c69b06dd4a0f 100644 --- a/tests/framework/utils/k8s_helper.go +++ b/tests/framework/utils/k8s_helper.go @@ -297,7 +297,7 @@ func (k8sh *K8sHelper) GetResource(args ...string) (string, error) { if err == nil { return result, nil } - return "", fmt.Errorf("Could Not get resource in k8s -- %v", err) + return result, fmt.Errorf("Could Not get resource in k8s -- %v", err) } func (k8sh *K8sHelper) CreateNamespace(namespace string) error { diff --git a/tests/integration/ceph_base_object_test.go b/tests/integration/ceph_base_object_test.go index e0620fb1da31..5f8779a2b2ef 100644 --- a/tests/integration/ceph_base_object_test.go +++ b/tests/integration/ceph_base_object_test.go @@ -23,6 +23,7 @@ import ( "os" "path/filepath" "strconv" + "strings" "testing" "time" @@ -35,6 +36,7 @@ import ( "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/wait" ) @@ -112,7 +114,7 @@ func createCephObjectStore(t *testing.T, helper *clients.TestClient, k8sh *utils // Check object store status t.Run("verify object store status", func(t *testing.T) { - retryCount := 30 + retryCount := 40 i := 0 for i = 0; i < retryCount; i++ { objectStore, err := k8sh.RookClientset.CephV1().CephObjectStores(namespace).Get(ctx, storeName, metav1.GetOptions{}) @@ -192,6 +194,12 @@ func assertObjectStoreDeletion(t *testing.T, k8sh *utils.K8sHelper, namespace, s sleepTime := 3 * time.Second for i = 0; i < retry; i++ { storeStr, err := k8sh.GetResource("-n", namespace, "CephObjectStore", storeName, "-o", "json") + // if cephobjectstore is not found, just return the test + // no need to check deletion phases as it is already deleted + if err != nil && strings.Contains(storeStr, errors.NewNotFound(v1.Resource("cephobjectstores.ceph.rook.io"), storeName).ErrStatus.Message) { + return + } + assert.NoError(t, err) logger.Infof("store: \n%s", storeStr) diff --git a/tests/integration/ceph_helm_test.go b/tests/integration/ceph_helm_test.go index a9f56c687f18..659488ed7fcd 100644 --- a/tests/integration/ceph_helm_test.go +++ b/tests/integration/ceph_helm_test.go @@ -71,7 +71,7 @@ func (h *HelmSuite) SetupSuite() { ChangeHostName: true, ConnectionsEncrypted: true, RookVersion: installer.LocalBuildTag, - CephVersion: installer.PacificVersion, + CephVersion: installer.QuincyVersion, } h.settings.ApplyEnvVars() h.installer, h.k8shelper = StartTestCluster(h.T, h.settings) diff --git a/tests/integration/ceph_multi_cluster_test.go b/tests/integration/ceph_multi_cluster_test.go index 0d4874de9680..96db2027f4cf 100644 --- a/tests/integration/ceph_multi_cluster_test.go +++ b/tests/integration/ceph_multi_cluster_test.go @@ -82,7 +82,7 @@ func (s *MultiClusterDeploySuite) SetupSuite() { MultipleMgrs: true, EnableAdmissionController: true, RookVersion: installer.LocalBuildTag, - CephVersion: installer.PacificVersion, + CephVersion: installer.QuincyVersion, RequireMsgr2: true, } s.settings.ApplyEnvVars() @@ -92,7 +92,7 @@ func (s *MultiClusterDeploySuite) SetupSuite() { Namespace: "multi-external", OperatorNamespace: s.settings.OperatorNamespace, RookVersion: s.settings.RookVersion, - CephVersion: installer.PacificVersion, + CephVersion: installer.QuincyVersion, } externalSettings.ApplyEnvVars() s.externalManifests = installer.NewCephManifests(externalSettings) diff --git a/tests/integration/ceph_upgrade_test.go b/tests/integration/ceph_upgrade_test.go index af9ff398255a..0383b65b389a 100644 --- a/tests/integration/ceph_upgrade_test.go +++ b/tests/integration/ceph_upgrade_test.go @@ -88,7 +88,7 @@ func (s *UpgradeSuite) baseSetup(useHelm bool, initialCephVersion v1.CephVersion Mons: 1, EnableDiscovery: true, SkipClusterCleanup: true, - RookVersion: installer.Version1_11, + RookVersion: installer.Version1_12, CephVersion: initialCephVersion, } @@ -97,11 +97,11 @@ func (s *UpgradeSuite) baseSetup(useHelm bool, initialCephVersion v1.CephVersion } func (s *UpgradeSuite) TestUpgradeRook() { - s.testUpgrade(false, installer.PacificVersion) + s.testUpgrade(false, installer.QuincyVersion) } func (s *UpgradeSuite) TestUpgradeHelm() { - s.testUpgrade(true, installer.PacificVersion) + s.testUpgrade(true, installer.QuincyVersion) } func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersionSpec) { @@ -127,9 +127,9 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi _ = s.helper.BucketClient.DeleteBucketStorageClass(s.namespace, installer.ObjectStoreName, installer.ObjectStoreSCName, "Delete") // - // Upgrade Rook from v1.11 to master + // Upgrade Rook from v1.12 to master // - logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_11) + logger.Infof("*** UPGRADING ROOK FROM %s to master ***", installer.Version1_12) s.gatherLogs(s.settings.OperatorNamespace, "_before_master_upgrade") s.upgradeToMaster() @@ -138,7 +138,7 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi err := s.installer.WaitForToolbox(s.namespace) assert.NoError(s.T(), err) - logger.Infof("Done with automatic upgrade from %s to master", installer.Version1_11) + logger.Infof("Done with automatic upgrade from %s to master", installer.Version1_12) newFile := "post-upgrade-previous-to-master-file" s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) rbdFilesToRead = append(rbdFilesToRead, newFile) @@ -150,24 +150,13 @@ func (s *UpgradeSuite) testUpgrade(useHelm bool, initialCephVersion v1.CephVersi // do not need retry b/c the OBC controller runs parallel to Rook-Ceph orchestration assert.True(s.T(), s.helper.BucketClient.CheckOBC(obcName, "bound")) - logger.Infof("Verified upgrade from %s to master", installer.Version1_11) + logger.Infof("Verified upgrade from %s to master", installer.Version1_12) // SKIP the Ceph version upgrades for the helm test if s.settings.UseHelm { return } - // - // Upgrade from pacific to quincy - // - logger.Infof("*** UPGRADING CEPH FROM PACIFIC TO QUINCY ***") - s.gatherLogs(s.settings.OperatorNamespace, "_before_quincy_upgrade") - s.upgradeCephVersion(installer.QuincyVersion.Image, numOSDs) - // Verify reading and writing to the test clients - newFile = "post-quincy-upgrade-file" - s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) - logger.Infof("Verified upgrade from pacific to quincy") - // // Upgrade from quincy to reef // @@ -215,12 +204,12 @@ func (s *UpgradeSuite) TestUpgradeCephToQuincyDevel() { checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false) } -func (s *UpgradeSuite) TestUpgradeCephToPacificDevel() { - s.baseSetup(false, installer.PacificVersion) +func (s *UpgradeSuite) TestUpgradeCephToReefDevel() { + s.baseSetup(false, installer.ReefVersion) objectUserID := "upgraded-user" preFilename := "pre-upgrade-file" - s.settings.CephVersion = installer.PacificVersion + s.settings.CephVersion = installer.ReefVersion numOSDs, rbdFilesToRead, cephfsFilesToRead := s.deployClusterforUpgrade(objectUserID, preFilename) clusterInfo := client.AdminTestClusterInfo(s.namespace) requireBlockImagesRemoved := false @@ -235,15 +224,15 @@ func (s *UpgradeSuite) TestUpgradeCephToPacificDevel() { }() // - // Upgrade from pacific to pacific devel + // Upgrade from reef to reef devel // - logger.Infof("*** UPGRADING CEPH FROM PACIFIC STABLE TO PACIFIC DEVEL ***") - s.gatherLogs(s.settings.OperatorNamespace, "_before_pacific_upgrade") - s.upgradeCephVersion(installer.PacificDevelVersion.Image, numOSDs) + logger.Infof("*** UPGRADING CEPH FROM REEF STABLE TO REEF DEVEL ***") + s.gatherLogs(s.settings.OperatorNamespace, "_before_reef_upgrade") + s.upgradeCephVersion(installer.ReefDevelVersion.Image, numOSDs) // Verify reading and writing to the test clients - newFile := "post-pacific-upgrade-file" + newFile := "post-reef-upgrade-file" s.verifyFilesAfterUpgrade(newFile, rbdFilesToRead, cephfsFilesToRead) - logger.Infof("verified upgrade from pacific stable to pacific devel") + logger.Infof("verified upgrade from reef stable to reef devel") checkCephObjectUser(&s.Suite, s.helper, s.k8sh, s.namespace, installer.ObjectStoreName, objectUserID, true, false) } @@ -297,7 +286,7 @@ func (s *UpgradeSuite) deployClusterforUpgrade(objectUserID, preFilename string) require.True(s.T(), created) // verify that we're actually running the right pre-upgrade image - s.verifyOperatorImage(installer.Version1_11) + s.verifyOperatorImage(installer.Version1_12) assert.NoError(s.T(), s.k8sh.WriteToPod("", rbdPodName, preFilename, simpleTestMessage)) assert.NoError(s.T(), s.k8sh.ReadFromPod("", rbdPodName, preFilename, simpleTestMessage)) diff --git a/tests/scripts/create-dev-cluster.sh b/tests/scripts/create-dev-cluster.sh index da17b3ab9416..fff7195b1443 100755 --- a/tests/scripts/create-dev-cluster.sh +++ b/tests/scripts/create-dev-cluster.sh @@ -1,21 +1,15 @@ #!/usr/bin/env bash -# Copyright 2021 The Rook Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -KUBECTL="minikube kubectl --" -ROOK_EXAMPLES_DIR="../../deploy/examples/" + +SCRIPT_ROOT=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P) +ROOK_EXAMPLES_DIR="${SCRIPT_ROOT}/../../deploy/examples/" + +init_vars(){ + local rook_profile_name=$1 + ROOK_PROFILE_NAME=$rook_profile_name + MINIKUBE="minikube --profile $rook_profile_name" + KUBECTL="$MINIKUBE kubectl --" +} wait_for_ceph_cluster() { echo "Waiting for ceph cluster" @@ -47,7 +41,8 @@ get_minikube_driver() { fi } -show_ceph_dashboard_info() { +show_info() { + local monitoring_enabled=$1 DASHBOARD_PASSWORD=$($KUBECTL -n rook-ceph get secret rook-ceph-dashboard-password -o jsonpath="{['data']['password']}" | base64 --decode && echo) IP_ADDR=$($KUBECTL get po --selector="app=rook-ceph-mgr" -n rook-ceph --output jsonpath='{.items[*].status.hostIP}') PORT="$($KUBECTL -n rook-ceph -o=jsonpath='{.spec.ports[?(@.name == "dashboard")].nodePort}' get services rook-ceph-mgr-dashboard-external-http)" @@ -56,25 +51,38 @@ show_ceph_dashboard_info() { echo "Ceph Dashboard: " echo " IP_ADDRESS: $BASE_URL" echo " PASSWORD: $DASHBOARD_PASSWORD" + if [ "$monitoring_enabled" = true ]; then + PROMETHEUS_API_HOST="http://$(kubectl -n rook-ceph -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0):30900" + echo "Prometheus Dashboard: " + echo " API_HOST: $PROMETHEUS_API_HOST" + fi echo "===========================" + echo " " + echo " *** To start using your rook cluster please set the following env: " + echo " " + echo " > eval \$($MINIKUBE docker-env)" + echo " > alias kubectl=\"$KUBECTL"\" + echo " " + echo " *** To access the new cluster with k9s: " + echo " " + echo " > k9s --context $ROOK_PROFILE_NAME" + echo " " } check_minikube_exists() { - minikube profile list > /dev/null 2>&1 - local retcode=$? - - if [ $retcode -eq 0 ]; then - echo "A minikube environment already exists, please use -f to force the cluster creation." + echo "Checking minikube profile '$ROOK_PROFILE_NAME'..." + if minikube profile list -l 2> /dev/null | grep -qE "\s$ROOK_PROFILE_NAME\s"; then + echo "A minikube profile '$ROOK_PROFILE_NAME' already exists, please use -f to force the cluster creation." exit 1 fi } setup_minikube_env() { minikube_driver="$(get_minikube_driver)" - echo "Setting up minikube env (using $minikube_driver driver)" - minikube delete - minikube start --disk-size=40g --extra-disks=3 --driver "$minikube_driver" - eval "$(minikube docker-env -p minikube)" + echo "Setting up minikube env for profile '$ROOK_PROFILE_NAME' (using $minikube_driver driver)" + $MINIKUBE delete + $MINIKUBE start --disk-size=40g --extra-disks=3 --driver "$minikube_driver" + eval "$($MINIKUBE docker-env)" } create_rook_cluster() { @@ -104,22 +112,44 @@ wait_for_rook_operator() { enable_rook_orchestrator() { echo "Enabling rook orchestrator" $KUBECTL rollout status deployment rook-ceph-tools -n rook-ceph --timeout=30s - kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph mgr module enable rook - kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch set backend rook - kubectl -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch status + $KUBECTL -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph mgr module enable rook + $KUBECTL -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch set backend rook + $KUBECTL -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph orch status +} + +enable_monitoring() { + echo "Enabling monitoring" + $KUBECTL apply -f https://raw.githubusercontent.com/coreos/prometheus-operator/v0.40.0/bundle.yaml + $KUBECTL wait --for=condition=ready pod -l app.kubernetes.io/name=prometheus-operator --timeout=30s + $KUBECTL apply -f monitoring/service-monitor.yaml + $KUBECTL apply -f monitoring/exporter-service-monitor.yaml + $KUBECTL apply -f monitoring/prometheus.yaml + $KUBECTL apply -f monitoring/prometheus-service.yaml + PROMETHEUS_API_HOST="http://$(kubectl -n rook-ceph -o jsonpath='{.status.hostIP}' get pod prometheus-rook-prometheus-0):30900" + $KUBECTL -n rook-ceph exec -it deploy/rook-ceph-tools -- ceph dashboard set-prometheus-api-host "$PROMETHEUS_API_HOST" } show_usage() { echo "" - echo " Usage: $(basename "$0") [-r] [-d /path/to/rook-examples/dir]" - echo " -r Enable rook orchestrator" - echo " -d value Path to Rook examples directory (i.e github.com/rook/rook/deploy/examples)" + echo " Usage: $(basename "$0") [-r] [-m] [-p ] [-d /path/to/rook-examples/dir]" + echo " -r Enable rook orchestrator" + echo " -m Enable monitoring" + echo " -p Specify the minikube profile name" + echo " -d value Path to Rook examples directory (i.e github.com/rook/rook/deploy/examples)" +} + +invocation_error() { + printf "%s\n" "$*" > /dev/stderr + show_usage + exit 1 } #################################################################### ################# MAIN ############################################# -while getopts ":hrfd:" opt; do + + +while getopts ":hrmfd:p:" opt; do case $opt in h) show_usage @@ -128,20 +158,23 @@ while getopts ":hrfd:" opt; do r) enable_rook=true ;; + m) + enable_monitoring=true + ;; f) force_minikube=true ;; d) ROOK_EXAMPLES_DIR="$OPTARG" ;; + p) + minikube_profile_name="$OPTARG" + ;; \?) - echo "Invalid option: -$OPTARG" >&2 - show_usage - exit 1 + invocation_error "Invalid option: -$OPTARG" ;; :) - echo "Option -$OPTARG requires an argument." >&2 - exit 1 + invocation_error "Option -$OPTARG requires an argument." ;; esac done @@ -150,6 +183,7 @@ echo "Using '$ROOK_EXAMPLES_DIR' as examples directory.." cd "$ROOK_EXAMPLES_DIR" || exit check_examples_dir +init_vars "${minikube_profile_name:-rook}" if [ -z "$force_minikube" ]; then check_minikube_exists @@ -164,7 +198,11 @@ if [ "$enable_rook" = true ]; then enable_rook_orchestrator fi -show_ceph_dashboard_info +if [ "$enable_monitoring" = true ]; then + enable_monitoring +fi + +show_info "$enable_monitoring" #################################################################### #################################################################### diff --git a/tests/scripts/github-action-helper.sh b/tests/scripts/github-action-helper.sh index b5dbd1f45177..664d47900f62 100755 --- a/tests/scripts/github-action-helper.sh +++ b/tests/scripts/github-action-helper.sh @@ -19,7 +19,7 @@ set -xeEo pipefail ############# # VARIABLES # ############# -: "${BLOCK:=$(sudo lsblk --paths | awk '/14G/ {print $1}' | head -1)}" +: "${BLOCK:=$(sudo lsblk --paths | awk '/14G/ || /64G/ {print $1}' | head -1)}" NETWORK_ERROR="connection reset by peer" SERVICE_UNAVAILABLE_ERROR="Service Unavailable" INTERNAL_ERROR="INTERNAL_ERROR" @@ -396,7 +396,7 @@ function create_LV_on_disk() { } function deploy_first_rook_cluster() { - BLOCK=$(sudo lsblk | awk '/14G/ {print $1}' | head -1) + BLOCK=$(sudo lsblk | awk '/14G/ || /64G/ {print $1}' | head -1) create_cluster_prerequisites cd deploy/examples/ @@ -409,7 +409,7 @@ function deploy_first_rook_cluster() { } function deploy_second_rook_cluster() { - BLOCK=$(sudo lsblk | awk '/14G/ {print $1}' | head -1) + BLOCK=$(sudo lsblk | awk '/14G/ || /64G/ {print $1}' | head -1) cd deploy/examples/ NAMESPACE=rook-ceph-secondary envsubst