From 6aae0a6cb4bbc1ba914c73c6e16d27351c18671f Mon Sep 17 00:00:00 2001 From: Sonny Bakker Date: Mon, 16 Sep 2024 12:13:53 +0200 Subject: [PATCH] [#13] Refactor after open-api-workflow d6a9fe1ff6c9ac6487839cda5907de6049d8aaec --- .github/workflows/ci.yml | 33 +++++++-------- .github/workflows/codeql.yml | 4 -- .../workflows/generate-postman-collection.yml | 7 ++++ .github/workflows/generate-sdks.yml | 7 ++++ .github/workflows/lint-oas.yml | 7 ++++ .github/workflows/oas-check.yml | 25 +++++++++++ .nvmrc | 1 + bin/check_schema.sh | 30 ------------- bin/compile_dependencies.sh | 9 ++-- bin/generate_postman_collection.sh | 22 ---------- bin/generate_sdks.sh | 42 ------------------- bin/lint_oas_files.sh | 16 ------- requirements/ci.in | 1 + requirements/ci.txt | 7 ++++ 14 files changed, 75 insertions(+), 136 deletions(-) create mode 100644 .github/workflows/oas-check.yml create mode 100644 .nvmrc delete mode 100755 bin/check_schema.sh delete mode 100755 bin/generate_postman_collection.sh delete mode 100755 bin/generate_sdks.sh delete mode 100755 bin/lint_oas_files.sh diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 654e3cf3..86065318 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,25 +19,25 @@ env: jobs: # determine changed files to decide if certain jobs can be skipped or not changed-files: - runs-on: ubuntu-latest # windows-latest | macos-latest + runs-on: ubuntu-latest name: Determine changed files steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v4 with: fetch-depth: 2 - name: Get changed PY files id: changed-py-files - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v45 with: - files: | - ^src/.+\.py + files: src/{,**/}*.py + - name: Get changed requirements files id: changed-requirements - uses: tj-actions/changed-files@v41 + uses: tj-actions/changed-files@v45 with: - files: ^requirements/.+\.txt$ + files: requirements/*.txt outputs: changed-py-files: ${{ steps.changed-py-files.outputs.any_changed }} @@ -68,19 +68,12 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - name: Set up backend environment + uses: maykinmedia/setup-django-backend@v1.3 with: python-version: '3.11' - - uses: actions/setup-node@v4 - with: - node-version: '18' + setup-node: true - - name: Install dependencies - run: pip install -r requirements/dev.txt codecov - - name: Build frontend - run: | - npm ci - npm run build - name: Run tests run: | python src/manage.py collectstatic --noinput --link @@ -103,7 +96,7 @@ jobs: cache: 'pip' cache-dependency-path: 'requirements/*.txt' - name: Install dependencies - run: pip install -r requirements/ci.txt pytest + run: pip install -r requirements/ci.txt - name: Generate environment variable documentation using OAf and check if it was updated run: | bin/generate_envvar_docs.sh @@ -119,7 +112,6 @@ jobs: runs-on: ubuntu-latest outputs: image-name: ${{ steps.image-name.outputs.image-name }} - sphinx-ssl-conf: ${{ steps.sphinx-ssl-conf.outputs.sphinx-ssl-conf }} steps: - run: echo "image-name=$IMAGE_NAME" >> $GITHUB_OUTPUT @@ -136,3 +128,6 @@ jobs: python-version: '3.11' docker-image-name: ${{ needs.store-reusable-workflow-vars.outputs.image-name }} repository-owner: 'maykinmedia' + secrets: + docker-username: ${{ secrets.DOCKER_USERNAME }} + docker-token: ${{ secrets.DOCKER_TOKEN }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 9fabd9e0..f89383ae 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -26,9 +26,5 @@ jobs: node-version: '18' postgres-version: 'latest' - isort-src-pattern: 'src' - black-src-pattern: 'src docs' - flake8-src-pattern: 'src' - django-settings-module: 'openklant.conf.ci' django-secret-key: dummy diff --git a/.github/workflows/generate-postman-collection.yml b/.github/workflows/generate-postman-collection.yml index 62e5245e..51cace59 100644 --- a/.github/workflows/generate-postman-collection.yml +++ b/.github/workflows/generate-postman-collection.yml @@ -12,3 +12,10 @@ on: jobs: open-api-workflow-generate-postman-collection: uses: maykinmedia/open-api-workflows/.github/workflows/generate-postman-collection.yml@initial-setup + strategy: + matrix: + component: + - contactgevens + - klantinteracties + with: + schema-path: 'src/openklant/components/${{ matrix.component }}/openapi.yaml' diff --git a/.github/workflows/generate-sdks.yml b/.github/workflows/generate-sdks.yml index 1e250a16..5a8c8266 100644 --- a/.github/workflows/generate-sdks.yml +++ b/.github/workflows/generate-sdks.yml @@ -12,3 +12,10 @@ on: jobs: open-api-workflow-generate-sdks: uses: maykinmedia/open-api-workflows/.github/workflows/generate-sdks.yml@initial-setup + strategy: + matrix: + component: + - contactgevens + - klantinteracties + with: + schema-path: 'src/openklant/components/${{ matrix.component }}/openapi.yaml' diff --git a/.github/workflows/lint-oas.yml b/.github/workflows/lint-oas.yml index bbdfe20e..794d1c61 100644 --- a/.github/workflows/lint-oas.yml +++ b/.github/workflows/lint-oas.yml @@ -12,3 +12,10 @@ on: jobs: open-api-workflow-lint-oas: uses: maykinmedia/open-api-workflows/.github/workflows/lint-oas.yml@initial-setup + strategy: + matrix: + component: + - contactgevens + - klantinteracties + with: + schema-path: 'src/openklant/components/${{ matrix.component }}/openapi.yaml' diff --git a/.github/workflows/oas-check.yml b/.github/workflows/oas-check.yml new file mode 100644 index 00000000..50965637 --- /dev/null +++ b/.github/workflows/oas-check.yml @@ -0,0 +1,25 @@ +name: check-oas + +on: + push: + paths: + - "src/openklant/components/*/openapi.yaml" + - .github/workflows/oas-check.yml + branches: + - '**' + workflow_dispatch: + +jobs: + open-api-workflow-check-oas: + uses: maykinmedia/open-api-workflows/.github/workflows/oas-check.yml@initial-setup + strategy: + matrix: + component: + - contactgegevens + - klantinteracties + with: + schema-path: 'src/openklant/components/${{ matrix.component }}/openapi.yaml' + schema-command: 'spectacular_for_component' + schema-options: "--component ${{ matrix.component }}" + python-version: '3.11' + django-settings-module: 'openklant.conf.ci' diff --git a/.nvmrc b/.nvmrc new file mode 100644 index 00000000..3c032078 --- /dev/null +++ b/.nvmrc @@ -0,0 +1 @@ +18 diff --git a/bin/check_schema.sh b/bin/check_schema.sh deleted file mode 100755 index cf94efd7..00000000 --- a/bin/check_schema.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -# Run this script from the root of the repository - -set -e - -COMPONENTS=( - contactgegevens - klantinteracties -) - -if [[ -z "$VIRTUAL_ENV" ]] && [[ ! -v GITHUB_ACTIONS ]]; then - echo "You need to activate your virtual env before running this script" - exit 1 -fi - -for component in "${COMPONENTS[@]}"; -do - ./bin/generate_schema_for_component.sh "$component" "openapi-$component.yaml" - - echo "Checking src/openklant/components/$component/openapi.yaml ..." - - diff "openapi-$component.yaml" "src/openklant/components/$component/openapi.yaml" - - if (( $? > 0 )); then - echo "src/openklant/components/$component/openapi.yaml needs to be updated!" - else - echo "src/openklant/components/$component/openapi.yaml is up-to-date." - fi -done diff --git a/bin/compile_dependencies.sh b/bin/compile_dependencies.sh index f1d03361..39c7fb6f 100755 --- a/bin/compile_dependencies.sh +++ b/bin/compile_dependencies.sh @@ -23,6 +23,8 @@ export CUSTOM_COMPILE_COMMAND="./bin/compile_dependencies.sh" # Base (& prod) deps pip-compile \ --no-emit-index-url \ + --output-file requirements/base.txt \ + --output-file requirements/production.txt \ "$@" \ requirements/base.in @@ -33,12 +35,13 @@ pip-compile \ "$@" \ requirements/base.txt \ requirements/test-tools.in \ - requirements/docs.in + requirements/ci.in # Dev depedencies - exact same set as CI + some extra tooling pip-compile \ --no-emit-index-url \ --output-file requirements/dev.txt \ "$@" \ - requirements/ci.txt \ - requirements/dev.in \ No newline at end of file + requirements/base.txt \ + requirements/test-tools.in \ + requirements/dev.in diff --git a/bin/generate_postman_collection.sh b/bin/generate_postman_collection.sh deleted file mode 100755 index 835ba9f0..00000000 --- a/bin/generate_postman_collection.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash - -# Run this script from the root of the repository - -set -e - -COMPONENTS=( - contactgegevens - klantinteracties -) - -OUTPUT_PATH="./tests/postman/collection.json" - -for component in "${COMPONENTS[@]}"; -do - echo "Converting src/openklant/components/$component/openapi.yaml into $OUTPUT_PATH ..." - - openapi2postmanv2 \ - --spec "./src/openklant/components/$component/openapi.yaml" \ - --output $OUTPUT_PATH \ - --pretty -done diff --git a/bin/generate_sdks.sh b/bin/generate_sdks.sh deleted file mode 100755 index 45bc4686..00000000 --- a/bin/generate_sdks.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash - -# Run this script from the root of the repository - -set -e - -COMPONENTS=( - contactgegevens - klantinteracties -) - -OUTPUT_DIR="./sdks" - - -for component in "${COMPONENTS[@]}"; -do - COMPONENT_PATH="./src/openklant/components/${component}/openapi.yaml" - - echo "Validating OAS for $COMPONENT_PATH" - openapi-generator-cli validate -i "${COMPONENT_PATH}" - - echo "Generating Java SDK for $component ..." - openapi-generator-cli generate -i "$COMPONENT_PATH" \ - --global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ - -o "$OUTPUT_DIR/java" \ - -g java \ - --additional-properties=dateLibrary=java8,java8=true,optionalProjectFile=false,optionalAssemblyInfo=false - - echo "Generating .NET SDK for $component ..." - openapi-generator-cli generate -i "$COMPONENT_PATH" \ - --global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ - -o "$OUTPUT_DIR/net" \ - -g csharp \ - --additional-properties=optionalProjectFile=false,optionalAssemblyInfo=false - - echo "Generating Python SDK for $component ..." - openapi-generator-cli generate -i "$COMPONENT_PATH" \ - --global-property=modelTests=false,apiTests=false,modelDocs=false,apiDocs=false \ - -o "$OUTPUT_DIR/python" \ - -g python \ - --additional-properties=optionalProjectFile=false,optionalAssemblyInfo=false+ -done diff --git a/bin/lint_oas_files.sh b/bin/lint_oas_files.sh deleted file mode 100755 index 8e4ba07d..00000000 --- a/bin/lint_oas_files.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -# Run this script from the root of the repository - -set -e - -COMPONENTS=( - contactgegevens - klantinteracties -) - -for component in "${COMPONENTS[@]}"; -do - echo "Linting src/openklant/components/$component/openapi.yaml ..." - spectral lint "src/openklant/components/$component/openapi.yaml" -done diff --git a/requirements/ci.in b/requirements/ci.in index 15f1c729..76375907 100644 --- a/requirements/ci.in +++ b/requirements/ci.in @@ -1 +1,2 @@ codecov +pytest diff --git a/requirements/ci.txt b/requirements/ci.txt index 1e968d5d..07e8a4f7 100644 --- a/requirements/ci.txt +++ b/requirements/ci.txt @@ -354,6 +354,8 @@ inflection==0.5.1 # -r requirements/base.txt # drf-spectacular # drf-yasg +iniconfig==2.0.0 + # via pytest iso-639==0.4.5 # via # -r requirements/base.txt @@ -438,6 +440,7 @@ packaging==24.1 # -r requirements/base.txt # black # drf-yasg + # pytest # sphinx pathspec==0.12.1 # via black @@ -451,6 +454,8 @@ platformdirs==4.2.2 # via # black # pylint +pluggy==1.5.0 + # via pytest prometheus-client==0.20.0 # via # -r requirements/base.txt @@ -494,6 +499,8 @@ pypng==0.20220715.0 # qrcode pyquery==2.0.0 # via -r requirements/test-tools.in +pytest==8.3.3 + # via -r requirements/ci.in python-dateutil==2.9.0.post0 # via # -r requirements/base.txt