From 9efd476d0a7b790570f1796de1e1df12195fd692 Mon Sep 17 00:00:00 2001 From: Mike Alfare Date: Wed, 17 Jul 2024 13:00:10 -0400 Subject: [PATCH] Move to `pre-commit` only (#777) * moving linting config into pre-commit-config * remove explicit call to mypy * run linting * fix enum reference --- .flake8 | 14 ---- .github/workflows/main.yml | 1 - .pre-commit-config.yaml | 102 +++++++++++---------------- dbt/adapters/redshift/connections.py | 13 ++-- dev-requirements.txt | 22 +++--- mypy.ini | 2 - tests/unit/utils.py | 1 + 7 files changed, 60 insertions(+), 95 deletions(-) delete mode 100644 .flake8 delete mode 100644 mypy.ini diff --git a/.flake8 b/.flake8 deleted file mode 100644 index bbc3202a0..000000000 --- a/.flake8 +++ /dev/null @@ -1,14 +0,0 @@ -[flake8] -select = - E - W - F -ignore = - # makes Flake8 work like black - W503, - W504, - # makes Flake8 work like black - E203, - E741, - E501, -exclude = test diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 238a191c9..2edb17f99 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -58,7 +58,6 @@ jobs: python -m pip install -r dev-requirements.txt python -m pip --version pre-commit --version - mypy --version dbt --version - name: pre-commit hooks diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3d80b955c..1ff672be8 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,63 +1,47 @@ -# For more on configuring pre-commit hooks (see https://pre-commit.com/) - -# Force all unspecified python hooks to run python 3.8 default_language_version: - python: python3 + python: python3 repos: -- repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 - hooks: - - id: check-yaml - args: [--unsafe] - - id: check-json - - id: end-of-file-fixer - - id: trailing-whitespace - - id: check-case-conflict -- repo: https://github.com/psf/black - rev: 23.1.0 - hooks: - - id: black - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - id: black - alias: black-check - stages: [manual] - additional_dependencies: ['click~=8.1'] - args: - - "--line-length=99" - - "--target-version=py38" - - "--check" - - "--diff" -- repo: https://github.com/pycqa/flake8 - rev: 6.0.0 - hooks: - - id: flake8 - - id: flake8 - alias: flake8-check - stages: [manual] -- repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.1.1 - hooks: - - id: mypy - # N.B.: Mypy is... a bit fragile. - # - # By using `language: system` we run this hook in the local - # environment instead of a pre-commit isolated one. This is needed - # to ensure mypy correctly parses the project. +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: check-yaml + args: [--unsafe] + - id: check-json + - id: end-of-file-fixer + - id: trailing-whitespace + - id: check-case-conflict + +- repo: https://github.com/psf/black + rev: 23.3.0 + hooks: + - id: black + args: + - --line-length=99 + - --target-version=py38 + +- repo: https://github.com/pycqa/flake8 + rev: 6.0.0 + hooks: + - id: flake8 + exclude: tests/ + args: + - --max-line-length=99 + - --select=E,F,W + - --ignore=E203,E501,E741,W503,W504 + - --per-file-ignores=*/__init__.py:F401 + additional_dependencies: [flaky] - # It may cause trouble in that it adds environmental variables out - # of our control to the mix. Unfortunately, there's nothing we can - # do about per pre-commit's author. - # See https://github.com/pre-commit/pre-commit/issues/730 for details. - args: [--show-error-codes, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters/.* - language: system - - id: mypy - alias: mypy-check - stages: [manual] - args: [--show-error-codes, --pretty, --ignore-missing-imports, --explicit-package-bases] - files: ^dbt/adapters - language: system +- repo: https://github.com/pre-commit/mirrors-mypy + rev: v1.2.0 + hooks: + - id: mypy + args: + - --show-error-codes + - --pretty + - --ignore-missing-imports + - --explicit-package-bases + files: ^dbt/adapters + additional_dependencies: + - types-pytz + - types-requests diff --git a/dbt/adapters/redshift/connections.py b/dbt/adapters/redshift/connections.py index 24c5aa1a5..e854a371e 100644 --- a/dbt/adapters/redshift/connections.py +++ b/dbt/adapters/redshift/connections.py @@ -10,6 +10,7 @@ from redshift_connector.utils.oids import get_datatype_name from dbt.adapters.sql import SQLConnectionManager +import dbt.clients.agate_helper from dbt.contracts.connection import AdapterResponse, Connection, Credentials from dbt.contracts.util import Replaceable from dbt.dataclass_schema import FieldEncoder, dbtClassMixin, StrEnum, ValidationError @@ -62,7 +63,7 @@ class UserSSLMode(StrEnum): @classmethod def default(cls) -> "UserSSLMode": # default for `psycopg2`, which aligns with dbt-redshift 1.4 and provides backwards compatibility - return cls.prefer + return cls("prefer") class RedshiftSSLMode(StrEnum): @@ -72,11 +73,11 @@ class RedshiftSSLMode(StrEnum): SSL_MODE_TRANSLATION = { UserSSLMode.disable: None, - UserSSLMode.allow: RedshiftSSLMode.verify_ca, - UserSSLMode.prefer: RedshiftSSLMode.verify_ca, - UserSSLMode.require: RedshiftSSLMode.verify_ca, - UserSSLMode.verify_ca: RedshiftSSLMode.verify_ca, - UserSSLMode.verify_full: RedshiftSSLMode.verify_full, + UserSSLMode.allow: RedshiftSSLMode("verify-ca"), + UserSSLMode.prefer: RedshiftSSLMode("verify-ca"), + UserSSLMode.require: RedshiftSSLMode("verify-ca"), + UserSSLMode.verify_ca: RedshiftSSLMode("verify-ca"), + UserSSLMode.verify_full: RedshiftSSLMode("verify-full"), } diff --git a/dev-requirements.txt b/dev-requirements.txt index 13ca5c02b..0bbcc2841 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -4,20 +4,14 @@ git+https://github.com/dbt-labs/dbt-core.git@1.5.latest#egg=dbt-core&subdirector git+https://github.com/dbt-labs/dbt-core.git@1.5.latest#egg=dbt-tests-adapter&subdirectory=tests/adapter git+https://github.com/dbt-labs/dbt-core.git@1.5.latest#egg=dbt-postgres&subdirectory=plugins/postgres -# if version 1.x or greater -> pin to major version -# if version 0.x -> pin to minor -black~=23.3 -bumpversion~=0.6.0 +# dev +ipdb~=0.13.13 +pre-commit==3.2.0 + +# test click~=8.1 ddtrace~=1.16 -flake8~=6.0 -flaky~=3.7 freezegun~=1.2 -ipdb~=0.13.13 -mypy==1.2.0 # patch updates have historically introduced breaking changes -pip-tools~=6.13 -pre-commit~=3.2 -pre-commit-hooks~=4.4 pytest~=7.3 pytest-csv~=3.0 pytest-dotenv~=0.5.2 @@ -25,7 +19,9 @@ pytest-logbook~=1.2 pytest-xdist~=3.2 pytz~=2023.3 tox~=4.5 -types-pytz~=2023.3 -types-requests~=2.28 + +# build +bumpversion~=0.6.0 +pip-tools~=6.13 twine~=4.0 wheel~=0.40 diff --git a/mypy.ini b/mypy.ini deleted file mode 100644 index b6e603581..000000000 --- a/mypy.ini +++ /dev/null @@ -1,2 +0,0 @@ -[mypy] -namespace_packages = True diff --git a/tests/unit/utils.py b/tests/unit/utils.py index f2ca418e3..32fb969b9 100644 --- a/tests/unit/utils.py +++ b/tests/unit/utils.py @@ -2,6 +2,7 @@ Note that all imports should be inside the functions to avoid import/mocking issues. """ + import string import os from unittest import mock