Merge branch 'cloudera-labs:devel' into devel

roles/rdbms/client/mysql_connector/meta/main.yml → .github/workflows/pre-commit.yml

@@ -1,15 +1,28 @@
-# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+# Copyright 2024 Cloudera, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
----
+name: Execute Precommit Linting and Checks
+
+on:
+  pull_request:
+  push:
+    branches: [main, devel]
+
+jobs:
+  pre-commit:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
+    - uses: pre-commit/[email protected]

.github/workflows/publish_docs.yml

@@ -19,8 +19,8 @@ name: Publish documentation
 on:
   push:
     branches:
-      - 'main' 
-  
+      - 'main'
+
   workflow_dispatch:
 
 jobs:

.github/workflows/validate_pr.yml

@@ -21,7 +21,7 @@ on:
     branches:
       - 'release/**'
       - 'devel'
-    
+
 jobs:
   validate:
     runs-on: ubuntu-latest
@@ -34,7 +34,7 @@ jobs:
         with:
           python-version: '3.9'
           cache: 'pip'
-      
+
       - name: Set up Ansible collections
         run: |
           sudo update-alternatives --install /usr/bin/python python $(which python3) 1
@@ -49,16 +49,16 @@ jobs:
           ansible --version
           ansible-galaxy collection list
           ansible-galaxy role list
-      
+
       - name: Set up Ansible collection dependencies
         run: |
           ansible-builder introspect --write-pip final_python.txt --write-bindep final_bindep.txt /usr/share/ansible/collections
           [[ -f final_python.txt ]] && pip install -r final_python.txt || echo "No Python dependencies found."
           [[ -f final_bindep.txt ]] && bindep --file final_bindep.txt || echo "No system dependencies found."
-      
+
       - name: Report installed Python dependencies
         run: pip freeze
-      
+
       - name: Validate collection
         run: |
           pushd /usr/share/ansible/collections/ansible_collections/cloudera/exe
@@ -75,7 +75,7 @@ jobs:
         run: |
           mkdir -p ./pr
           echo $PR_NUMBER > ./pr/pr_number
-      
+
       - name: Upload the PR number
         uses: actions/upload-artifact@v3
         with:

.github/workflows/validate_pr_docs.yml

@@ -21,7 +21,7 @@ on:
     branches:
       - 'release/**'
       - 'devel'
-    
+
   workflow_dispatch:
 
 jobs:

.gitignore

@@ -161,3 +161,6 @@ dmypy.json
 
 # Cython debug symbols
 cython_debug/
+
+# UV package manager
+uv.lock

roles/rdbms/client/oracle_connector/meta/main.yml → .pre-commit-config.yaml

@@ -1,15 +1,26 @@
-# Copyright 2023 Cloudera, Inc. All Rights Reserved.
+# Copyright 2024 Cloudera, Inc.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
 #
-#      http://www.apache.org/licenses/LICENSE-2.0
+#     https://www.apache.org/licenses/LICENSE-2.0
 #
 # Unless required by applicable law or agreed to in writing, software
 # distributed under the License is distributed on an "AS IS" BASIS,
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
----
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v3.2.0
+    hooks:
+      - id: trailing-whitespace
+      - id: end-of-file-fixer
+      - id: check-yaml
+      - id: check-added-large-files
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
+    hooks:
+      - id: black

CONTRIBUTING.md

@@ -4,7 +4,7 @@ Thank you for considering contributions to the `cloudera.exe` Ansible collection
 
 ## Submitting a pull request
 
-You can start work on issues that are not yet part of a [Milestone](https://github.com/cloudera-labs/cloudera.exe/milestones) -- anything in our issue tracker that isn't assigned to a Milestone is considered the [backlog](https://github.com/cloudera-labs/cloudera.exe/issues?q=is%3Aopen+is%3Aissue+no%3Amilestone). 
+You can start work on issues that are not yet part of a [Milestone](https://github.com/cloudera-labs/cloudera.exe/milestones) -- anything in our issue tracker that isn't assigned to a Milestone is considered the [backlog](https://github.com/cloudera-labs/cloudera.exe/issues?q=is%3Aopen+is%3Aissue+no%3Amilestone).
 
 Before you start working, please announce that you want to do so by commenting on the issue. _([Create an issue](https://github.com/cloudera-labs/cloudera.exe/issues/new?labels=enhancement) if there isn't one yet, and you can also check out our [Discussions](https://github.com/cloudera-labs/cloudera.exe/discussions) for ideas.)_ We try to ensure that all active work is assigned to a Milestone in order to keep our backlog accurate.
 
@@ -21,6 +21,16 @@ As a _best practice_, you can prefix your branches with:
 > [!NOTE]
 > :fire_extinguisher: A **hotfix** should branch from `main`. It will then be committed to both the `main` and `devel` branches.
 
+## Running pre-commit
+
+We use [pre-commit](https://pre-commit.com/) to lint and validate submissions.
+
+Setting up `pre-commit` is straightforward; see the [Installation](https://pre-commit.com/#installation) documentation on the `pre-commit` site for details.
+
+We highly suggest you install the local `git` hook scripts as well; run `pre-commit install` within the root of the project.
+
+Lastly, you can run the hooks prior to committing changes (and PR submission) by executing `pre-commit run -a`.
+
 ## Signing your commits
 
 Note that we require signed commits inline with [Developer Certificate of Origin](https://developercertificate.org/) best-practices for open source collaboration.

LICENSE

@@ -198,4 +198,4 @@ Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
-limitations under the License.
+limitations under the License.

README.md

@@ -36,7 +36,7 @@ The other collection assets - the _roles_ and _plugins_ - are detailed in the [A
 
 ## API
 
-See the [API documentation](https://cloudera-labs.github.io/cloudera.exe/) for details for each plugin and role within the collection. 
+See the [API documentation](https://cloudera-labs.github.io/cloudera.exe/) for details for each plugin and role within the collection.
 
 ## Roadmap
 
@@ -123,7 +123,7 @@ See the [requirements for `cloudera-labs/cloudera.cluster`](https://github.com/w
 
 **Terraform**
 
-If you intend to use Terraform as your infrastructure engine within the `cloudera.exe.infra` role, then install the following: 
+If you intend to use Terraform as your infrastructure engine within the `cloudera.exe.infra` role, then install the following:
 
 | Name | Type | Version |
 |------|------|---------|
@@ -220,7 +220,7 @@ For more information on the collection, check out:
 To create a local collection tarball, run:
 
 ```bash
-ansible-galaxy collection build 
+ansible-galaxy collection build
 ```
 
 ## Building the API Documentation

builder/requirements.yml

@@ -17,7 +17,7 @@
 collections:
   - source: .
     type: dir
-    
+
 roles:
   - name: geerlingguy.postgresql
     version: 3.5.0

docs/configuration.md

@@ -1,4 +1,4 @@
 # Configuration Variables
 
 Please see the [configuration.yml](./configuration.yml) for current parameters.
-Detailed explanations are forthcoming.
+Detailed explanations are forthcoming.

docs/configuration.yml

@@ -121,7 +121,7 @@ dw:
     template:
   dbc:
     suffix:
-    default_suffix: 
+    default_suffix:
   tags:
   overlay_network:
   private_load_balancer:
@@ -189,15 +189,15 @@ env:
         identity:
         log:
         ranger_audit:
-        raz: 
+        raz:
         xaccount:
       name:
         cross_account:
         datalake_admin:
         idbroker:
         log:
         ranger_audit:
-        raz: 
+        raz:
       name_suffix:
         admin:
         assignment:
@@ -242,7 +242,7 @@ env:
     user_sync:
     version:
     scale:
-    enable_raz: 
+    enable_raz:
   gcp:
     auto_enable_services:
     availability_zones:
@@ -377,12 +377,12 @@ infra:
       tags:
     storage:
       tags:
-      bucket_object_ownership: 
-      bucket_acl_permissions: 
+      bucket_object_ownership:
+      bucket_acl_permissions:
     private_endpoints:
     prefix_list:
       suffix:
-    extra_prefix_list_name: 
+    extra_prefix_list_name:
   azure:
     metagroup:
       name:
@@ -506,4 +506,4 @@ data:
       suffix:
   teardown:
       delete_policies:
-      delete_roles: 
+      delete_roles:

docs/design.md

@@ -12,7 +12,7 @@ Other roles use the shared dependencies and tagging approach, yet are not direct
 
 # Task Lists
 
-Each of the three core roles (`infrastructure`, `platform`, and `runtime`) consist of four entrypoint task lists: 
+Each of the three core roles (`infrastructure`, `platform`, and `runtime`) consist of four entrypoint task lists:
 
 + `validate.yml`
 + `initialize.yml`
@@ -23,7 +23,7 @@ These tasks lists are referenced by the `ansible.builtin.include_role` module's
 
 # Variables
 
-The variables used by any `cloudera.exe` role are defined _completely_ within the role's `defaults` and prefixed to isolate variables to that role alone. Roles also reference a [canonical set of variables](configuration.md) to normalize usage. (See [Configuration Mapping](#configuration-mapping) for customization options). 
+The variables used by any `cloudera.exe` role are defined _completely_ within the role's `defaults` and prefixed to isolate variables to that role alone. Roles also reference a [canonical set of variables](configuration.md) to normalize usage. (See [Configuration Mapping](#configuration-mapping) for customization options).
 
 Variables that are shared among roles should be defined within the `cloudera.exe.common` role, which is a dependency for each of the core roles, and then mapped into a prefixed role default.
 
@@ -44,9 +44,9 @@ This mapping isolates changes to shared variables to a single location (`clouder
 
 # Tags
 
-The core roles - `infrastructure`, `platform`, and `runtime` -- are designed to be invoked by [Ansible tags](https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html). 
+The core roles - `infrastructure`, `platform`, and `runtime` -- are designed to be invoked by [Ansible tags](https://docs.ansible.com/ansible/latest/user_guide/playbooks_tags.html).
 
-Roles like `cloudera.exe.sequence` have a tagging structure which "cascades" execution to provide runlevel prerequisites. For example, the `plat` tag (representing the [Platform](runlevels.md#platform) runlevel) invokes the _teardown_ sequence for [Runtime](runlevels.md#runtime) and then the _setup_ sequence for [Infrastructure](runlevels.md#infrastructure) and then Platform itself, reconciling the state of each setup if needed. 
+Roles like `cloudera.exe.sequence` have a tagging structure which "cascades" execution to provide runlevel prerequisites. For example, the `plat` tag (representing the [Platform](runlevels.md#platform) runlevel) invokes the _teardown_ sequence for [Runtime](runlevels.md#runtime) and then the _setup_ sequence for [Infrastructure](runlevels.md#infrastructure) and then Platform itself, reconciling the state of each setup if needed.
 
 The _validate_ sequence for the [Validation](runlevels.md#validation) runlevel is always invoked, as is the _initialization_ sequence -- the [Initialization](runlevels.md#initialization) for each affected high-level runlevel.
 
@@ -82,12 +82,12 @@ globals:
   tasks:
     - name: Import the core Runlevels
       ansible.builtin.import_role:
-        name: cloudera.exe.sequence 
+        name: cloudera.exe.sequence
 ```
 
 ## Tag Extension
 
-Playbook developers that import the `cloudera.exe.sequence` role (or other future "orchestration" roles) can add both additional and existing "runlevel" tags to invoke other roles or tasks. This technique is well-suited for extending the core to handle [additional runlevels](#additional-runlevels). 
+Playbook developers that import the `cloudera.exe.sequence` role (or other future "orchestration" roles) can add both additional and existing "runlevel" tags to invoke other roles or tasks. This technique is well-suited for extending the core to handle [additional runlevels](#additional-runlevels).
 
 For example, to add a pre-deploy role:
 
@@ -125,7 +125,7 @@ For example, to add a pre-deploy role:
         - dh
     - name: Import the core Runlevels (and their tags)
       ansible.builtin.import_role:
-        name: cloudera.exe.sequence 
+        name: cloudera.exe.sequence
 ```
 
 This example above can execute just the _pre-setup_ role:
@@ -180,7 +180,7 @@ For example, adding an explicit `install` tag to execute the [Installation](runl
         - dw
         - opdb
         - dh
-    
+
     - name: Validate Platform Configuration
       ansible.builtin.include_role:
         name: cloudera.exe.platform
@@ -245,4 +245,4 @@ For example, adding an explicit `install` tag to execute the [Installation](runl
 
 ## Dependencies
 
-Additional runlevels commonly have a dependency on the `cloudera.exe.common` role; in the example above, the `my_namespace.my_collection.my_app_role/meta/main.yml` could have an entry in the `dependencies` parameter referencing `cloudera.exe.common`.
+Additional runlevels commonly have a dependency on the `cloudera.exe.common` role; in the example above, the `my_namespace.my_collection.my_app_role/meta/main.yml` could have an entry in the `dependencies` parameter referencing `cloudera.exe.common`.

docs/runlevels.md

@@ -45,7 +45,7 @@ Destroys all cloud provider and CDP assets, subject to configuration allowances.
 The runlevel is invoked with the `teardown` tag.
 
 ## Validation
-Checks the incoming configuration parameters for errors and misalignment. No external interaction. This runlevel is in scope of the `cloudera.exe` collection. 
+Checks the incoming configuration parameters for errors and misalignment. No external interaction. This runlevel is in scope of the `cloudera.exe` collection.
 
 The runlevel is invoked with the `validate` tag.
 
@@ -55,7 +55,7 @@ Checks and initializes authentication and authorization for the cloud and VM con
 The runlevel is not directly invocable.
 
 ## Infrastructure
-Manages cloud-provider resources, VM control planes, instance OS and services management, networking, etc. This runlevel is in scope of the `cloudera.exe` collection. 
+Manages cloud-provider resources, VM control planes, instance OS and services management, networking, etc. This runlevel is in scope of the `cloudera.exe` collection.
 
 The runlevel is invoked with the `infra` tag.
 
@@ -72,7 +72,7 @@ The runlevel is invoked with the `run` tag.
 ## Installation
 Executes post-runtime and platform configurations. This runlevel is outside the scope of the `cloudera.exe` collection. Actual runtime projects should define or import additional runlevel roles and/or tasks as required.
 
-The suggested tag for this runlevel is `install`. 
+The suggested tag for this runlevel is `install`.
 
 ## Application
 Executes user- and application-specific tasks, e.g. table definitions, security policy configurations, data loading and management. This runlevel is outside the scope of the `cloudera.exe` collection. Actual runtime projects should define or import additional runlevel roles and/or tasks as required.
@@ -82,4 +82,4 @@ The suggested tag for this runlevel is `app`.
 ## Operations
 Executes management tasks broadly. This runlevel is outside the scope of the `cloudera.exe` collection. Actual runtime projects should define or import additional runlevel roles and/or tasks as required.
 
-There are no suggested tags for this runlevel.
+There are no suggested tags for this runlevel.

docsbuild/.gitignore

@@ -3,4 +3,4 @@
 
 /temp-rst
 /build
-/rst
+/rst

docsbuild/antsibull-docs.cfg

@@ -47,4 +47,4 @@ logging_cfg = {
       output_name = stderr
     }
   }
-}
+}