From 3c18a61640ec114ed3574aa1c2e0569c7b2a1943 Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Tue, 9 Jul 2024 18:32:33 +0200 Subject: [PATCH 1/3] Build multiplatform image for amd64 and arm64 architectures on release --- .github/workflows/continuous-integration.yml | 217 +++--------------- .github/workflows/pr-cleanup.yml | 51 ++++ ...sdk-dev-docker-image.yml => pr-merged.yml} | 28 +-- .github/workflows/release-artifacts.yml | 161 +++++++++++++ Dockerfile | 8 +- Makefile | 62 ++--- docker-bake.hcl | 12 - 7 files changed, 291 insertions(+), 248 deletions(-) create mode 100644 .github/workflows/pr-cleanup.yml rename .github/workflows/{rasa-sdk-dev-docker-image.yml => pr-merged.yml} (63%) create mode 100644 .github/workflows/release-artifacts.yml delete mode 100644 docker-bake.hcl diff --git a/.github/workflows/continuous-integration.yml b/.github/workflows/continuous-integration.yml index 0f8e5c60d..f4301719b 100644 --- a/.github/workflows/continuous-integration.yml +++ b/.github/workflows/continuous-integration.yml @@ -10,11 +10,18 @@ on: env: IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }} + DEV_REPOSITORY: 329710836760.dkr.ecr.us-east-1.amazonaws.com/rasa-sdk-dev + AWS_REGION: us-east-1 # SECRETS # - PYPI_TOKEN: publishing token for amn41 account, needs to be maintainer of # RasaHQ/rasa-sdk on pypi (account credentials in 1password) +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + + jobs: quality: name: Code Quality @@ -128,204 +135,34 @@ jobs: with: dockerfile: "Dockerfile" - build_docker_image_set_env: - name: Prepare environment for Docker build - runs-on: ubuntu-22.04 - outputs: - # Tag name used for intermediate images created during Docker image builds, e.g. 3886 - a PR number - image_tag: ${{ steps.set_output.outputs.image_tag }} - # Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version - is_newest_version: ${{ steps.rasa_sdk_get_version.outputs.is_newest_version }} - steps: - # Due to an issue with checking out a wrong commit, we make sure - # to checkout HEAD commit for a pull request. - # More details: https://github.com/actions/checkout/issues/299 - - name: Checkout pull request HEAD commit instead of merge commit 🕝 - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 - if: github.event_name == 'pull_request' - with: - ref: ${{ github.event.pull_request.head.sha }} - - - name: Checkout git repository 🕝 - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 - if: github.event_name != 'pull_request' - - # Set environment variables for a pull request - # - # In this scenario, we've created a PR #1234 - # - # Example output: - # IMAGE_TAG=1234 - - name: Set environment variables - pull_request - if: github.event_name == 'pull_request' && env.IS_TAG_BUILD == 'false' - run: | - echo "IMAGE_TAG=${{ github.event.number }}" >> $GITHUB_ENV - - # Set environment variables for a tag - # - # In this scenario, we've pushed the '2.4.1' tag - # - # Example output: - # IMAGE_TAG=2.4.1 - - name: Set environment variables - push - tag - if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' - run: | - TAG_NAME=${GITHUB_REF#refs/tags/} - echo "IMAGE_TAG=${TAG_NAME}" >> $GITHUB_ENV - - # Set environment variables for a branch - # - # In this scenario, we've pushed changes into the main branch - # - # Example output: - # IMAGE_TAG=main - - name: Set environment variables - push - branch - if: github.event_name == 'push' && env.IS_TAG_BUILD == 'false' - run: | - BRANCH_NAME=${GITHUB_REF#refs/heads/} - echo "IMAGE_TAG=${BRANCH_NAME}" >> $GITHUB_ENV - - - name: Set output - id: set_output - run: | - echo "image_tag=${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT - - - name: Check if tag version is equal or higher than the latest tagged rasa-sdk version - id: rasa_sdk_get_version - if: env.IS_TAG_BUILD == 'true' - run: | - # Get latest tagged rasa-sdk version - git fetch --depth=1 origin "+refs/tags/*:refs/tags/*" - # Fetch branch history - git fetch --prune --unshallow - LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION=$(git tag | sort -r -V | grep -E "^[0-9.]+$" | head -n1) - CURRENT_TAG=${GITHUB_REF#refs/tags/} - # Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version - IS_NEWEST_VERSION=$((printf '%s\n%s\n' "${LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION}" "$CURRENT_TAG" \ - | sort -V -C && echo true || echo false) || true) - - - if [[ "${IS_NEWEST_VERSION}" == "true" && "$CURRENT_TAG" =~ ^[0-9.]+$ ]]; then - echo "is_newest_version=true" >> $GITHUB_OUTPUT - else - echo "is_newest_version=false" >> $GITHUB_OUTPUT - fi - - build_docker_image: - name: Build Docker image + rasa-sdk-dev-docker-image: + name: Build dev Docker image runs-on: ubuntu-22.04 - needs: [quality, test, docker_linter, build_docker_image_set_env] steps: - # Due to an issue with checking out a wrong commit, we make sure - # to checkout HEAD commit for a pull request. - # More details: https://github.com/actions/checkout/issues/299 - - name: Checkout pull request HEAD commit instead of merge commit 🕝 - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 - if: github.event_name == 'pull_request' - with: - ref: ${{ github.event.pull_request.head.sha }} + - name: Check out code + uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Checkout git repository 🕝 - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 - if: github.event_name != 'pull_request' - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1 with: - version: v0.5.1 - driver: docker + role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }} + aws-region: ${{ env.AWS_REGION }} - - name: Set environment variables - run: | - echo "IMAGE_TAG=${{ needs.build_docker_image_set_env.outputs.image_tag }}" >> $GITHUB_ENV - - - name: Login to Docker Hub Registry 🔢 - if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa-sdk' - run: echo ${{ secrets.DOCKER_HUB_PASSWORD }} | docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} --password-stdin || true - - - name: Build and Push Docker image 📦 - if: github.event_name == 'push' && env.IS_TAG_BUILD == 'true' && github.repository == 'RasaHQ/rasa-sdk' - run: | - IS_NEWEST_VERSION=${{ needs.build_docker_image_set_env.outputs.is_newest_version }} - - # Push image - IMAGE_TAG=${IMAGE_TAG} docker buildx bake --set *.platform=linux/amd64,linux/arm64 -f docker-bake.hcl default --push - - # Tag the image as latest - if [[ "${IS_NEWEST_VERSION}" == "true" ]]; then - docker tag rasa/rasa-sdk:${IMAGE_TAG} rasa/rasa-sdk:latest - docker push rasa/rasa-sdk:latest - fi - - deploy: - name: Deploy to PyPI - runs-on: ubuntu-22.04 - - # deploy will only be run when there is a tag available - if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags') - needs: [quality, test, build_docker_image] # only run after all other stages succeeded - - steps: - - name: Checkout git repository 🕝 - uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 - - - name: Set up Python 3.10 🐍 - uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 with: - python-version: '3.10' + mask-password: "true" - - name: Read Poetry Version 🔢 - run: | - echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV - shell: bash + - name: Set up QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 - - name: Install poetry 🦄 - uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 - with: - poetry-version: ${{ env.POETRY_VERSION }} - - - name: Build ⚒️ Distributions - run: poetry build - - - name: Publish to PyPI 📦 - uses: pypa/gh-action-pypi-publish@bea5cda687c2b79989126d589ef4411bedce0195 - with: - user: __token__ - password: ${{ secrets.PYPI_TOKEN }} - skip_existing: true + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 - - name: Publish Release Notes 🗞 - if: env.IS_TAG_BUILD - env: - GITHUB_TAG: ${{ github.ref }} - GITHUB_REPO_SLUG: ${{ github.repository }} - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Build and push docker image to AWS run: | - GITHUB_TAG=${GITHUB_TAG/refs\/tags\//} - sudo apt-get update - sudo apt-get -y install pandoc - pip install -U github3.py pep440_version_utils - python3 ${GITHUB_WORKSPACE}/scripts/publish_gh_release_notes.py - - - name: Get RASA SDK Version - env: - RASA_SDK_VERSION: ${{ github.ref }} - run: | - echo "RASA_SDK_VERSION=${RASA_SDK_VERSION/refs\/tags\//}" >> $GITHUB_ENV - - - name: Notify Slack 💬 - if: env.IS_TAG_BUILD && success() - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }} - uses: Ilshidur/action-slack@2.1.0 - with: - args: "⚡ New *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} has been released! Changelog: https://github.com/RasaHQ/rasa-sdk/blob/${{ env.RASA_SDK_VERSION }}/CHANGELOG.mdx" - - - name: Notify Slack of Failure ⛔ - if: env.IS_TAG_BUILD && failure() - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }} - uses: Ilshidur/action-slack@2.1.0 - with: - args: "⛔️ *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} could not be released 😱 GitHub Actions: https://github.com/RasaHQ/rasa-sdk/actions?query=branch%3A${{ env.RASA_SDK_VERSION }}" \ No newline at end of file + IMAGE_NAME=${{ env.DEV_REPOSITORY }} \ + IMAGE_TAG=pr${{ github.event.number }} \ + make build-and-push-multi-platform-docker \ No newline at end of file diff --git a/.github/workflows/pr-cleanup.yml b/.github/workflows/pr-cleanup.yml new file mode 100644 index 000000000..8d8346058 --- /dev/null +++ b/.github/workflows/pr-cleanup.yml @@ -0,0 +1,51 @@ +name: CI On Pull Request Closure +on: + pull_request: + types: [closed] + +env: + AWS_REGION: us-east-1 + +permissions: + id-token: write # This is required for requesting the JWT + contents: read # This is required for actions/checkout + +jobs: + aws-ecr: + # Skip any PR created by dependabot to avoid permission issues + if: (github.actor != 'dependabot[bot]') + name: Delete PR Docker Images + runs-on: ubuntu-22.04 + steps: + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 #v3.0.1 + with: + role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }} + aws-region: ${{ env.AWS_REGION }} + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@2fc7aceee09e9e4a7105c0d060c656fad0b4f63d #v1.6 + with: + mask-password: "true" + + - name: Remove docker images from all ECR repositories + run: | + REPO_NAMES=("rasa-sdk-dev") + SEARCH_STRING="pr${{ github.event.number }}" + # Iterate through each repository + for REPO_NAME in "${REPO_NAMES[@]}" + do + # List all image tags in the repository that match the search string + # Escape empty strings and null character + IMAGES=$(aws ecr list-images --repository-name $REPO_NAME --query "imageIds[?imageTag!=null&&imageTag!=''&&contains(imageTag, '$SEARCH_STRING')].imageTag" --output text) + + # Print the image tags + echo "Images to be deleted in repository $REPO_NAME:" + echo "$IMAGES" + # Delete each image by its tag + for IMAGE in $IMAGES + do + aws ecr batch-delete-image --repository-name $REPO_NAME --image-ids imageTag=$IMAGE + done + done \ No newline at end of file diff --git a/.github/workflows/rasa-sdk-dev-docker-image.yml b/.github/workflows/pr-merged.yml similarity index 63% rename from .github/workflows/rasa-sdk-dev-docker-image.yml rename to .github/workflows/pr-merged.yml index 49ffb5f4f..25bcbb27d 100644 --- a/.github/workflows/rasa-sdk-dev-docker-image.yml +++ b/.github/workflows/pr-merged.yml @@ -1,11 +1,9 @@ -name: Create Dev Docker Images +name: Pull Request Merged to Main on: - schedule: - # Run cron job at 8AM Monday to Sunday. - - cron: '0 8 * * *' - workflow_dispatch: - pull_request: + push: + branches: + - main env: AWS_REGION: us-east-1 @@ -21,18 +19,13 @@ permissions: jobs: rasa-sdk-dev-docker-image: - name: rasa-sdk dev docker image + name: Build Dev Docker Image and Push to AWS runs-on: ubuntu-22.04 steps: - name: Check out code uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Read Poetry Version 🔢 - run: | - echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV - shell: bash - - name: Configure AWS credentials uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1 with: @@ -45,11 +38,14 @@ jobs: with: mask-password: "true" + - name: Set up QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + - name: Set up Docker Buildx - uses: docker/setup-buildx-action@4b4e9c3e2d4531116a6f8ba8e71fc6e2cb6e6c8c - with: - version: v0.5.1 + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 - name: Build and push docker image to AWS run: | - docker buildx bake --set *.platform=linux/amd64,linux/arm64 --set default.tags=$REPOSITORY:latest --push + IMAGE_NAME=${{ env.REPOSITORY }} \ + IMAGE_TAG=latest \ + make build-and-push-multi-platform-docker diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml new file mode 100644 index 000000000..d31ff31ac --- /dev/null +++ b/.github/workflows/release-artifacts.yml @@ -0,0 +1,161 @@ +name: Release artifacts + +on: + push: + tags: + - "*" + +jobs: + build_docker_image_set_env: + name: Prepare environment for Docker build + runs-on: ubuntu-22.04 + if: github.repository == 'RasaHQ/rasa-sdk' + outputs: + # Tag name used for intermediate images created during Docker image builds, e.g. 3886 - a PR number + image_tag: ${{ steps.set_output.outputs.image_tag }} + # Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version + is_newest_version: ${{ steps.rasa_sdk_get_version.outputs.is_newest_version }} + steps: + - name: Checkout git repository 🕝 + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + + # Set environment variables for a tag + # + # In this scenario, we've pushed the '2.4.1' tag + # + # Example output: + # IMAGE_TAG=2.4.1 + - name: Set image_tag + run: | + TAG_NAME=${GITHUB_REF#refs/tags/} + echo "IMAGE_TAG=${TAG_NAME}" >> $GITHUB_ENV + echo "image_tag=${{ env.IMAGE_TAG }}" >> $GITHUB_OUTPUT + + - name: Set is_newest_version + id: rasa_sdk_get_version + run: | + # Get latest tagged rasa-sdk version + git fetch --depth=1 origin "+refs/tags/*:refs/tags/*" + # Fetch branch history + git fetch --prune --unshallow + LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION=$(git tag | sort -r -V | grep -E "^[0-9.]+$" | head -n1) + CURRENT_TAG=${GITHUB_REF#refs/tags/} + # Return 'true' if tag version is equal or higher than the latest tagged rasa-sdk version + IS_NEWEST_VERSION=$((printf '%s\n%s\n' "${LATEST_TAGGED_NON_ALPHA_RASA_SDK_VERSION}" "$CURRENT_TAG" \ + | sort -V -C && echo true || echo false) || true) + + + if [[ "${IS_NEWEST_VERSION}" == "true" && "$CURRENT_TAG" =~ ^[0-9.]+$ ]]; then + echo "is_newest_version=true" >> $GITHUB_OUTPUT + else + echo "is_newest_version=false" >> $GITHUB_OUTPUT + fi + + build_docker_image: + name: Build Docker image + runs-on: ubuntu-22.04 + needs: [ build_docker_image_set_env ] + if: github.repository == 'RasaHQ/rasa-sdk' + + steps: + - name: Checkout git repository 🕝 + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + + - name: Set up QEMU + uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c # v3.1.0 + + - name: Set environment variables + run: | + echo "IMAGE_TAG=${{ needs.build_docker_image_set_env.outputs.image_tag }}" >> $GITHUB_ENV + + - name: Login to Docker Hub Registry 🔢 + run: echo ${{ secrets.DOCKER_HUB_PASSWORD }} | docker login -u ${{ secrets.DOCKER_HUB_USERNAME }} --password-stdin || true + + - name: Build and Push Docker image 📦 + run: | + IS_NEWEST_VERSION=${{ needs.build_docker_image_set_env.outputs.is_newest_version }} + + # Push image + IMAGE_NAME=rasa/rasa-sdk \ + IMAGE_TAG=${IMAGE_TAG} \ + make build-and-push-multi-platform-docker + + # Tag the image as latest + if [[ "${IS_NEWEST_VERSION}" == "true" ]]; then + IMAGE_TAG=latest \ + make build-and-push-multi-platform-docker + fi + + deploy: + name: Deploy to PyPI + runs-on: ubuntu-22.04 + + # deploy will only be run when there is a tag available + needs: [ build_docker_image ] # only run after all other stages succeeded + + steps: + - name: Checkout git repository 🕝 + uses: actions/checkout@8f4b7f84864484a7bf31766abe9204da3cbe65b3 + + - name: Set up Python 3.10 🐍 + uses: actions/setup-python@d27e3f3d7c64b4bbf8e4abfb9b63b83e846e0435 + with: + python-version: '3.10' + + - name: Read Poetry Version 🔢 + run: | + echo "POETRY_VERSION=$(scripts/poetry-version.sh)" >> $GITHUB_ENV + shell: bash + + - name: Install poetry 🦄 + uses: Gr1N/setup-poetry@15821dc8a61bc630db542ae4baf6a7c19a994844 + with: + poetry-version: ${{ env.POETRY_VERSION }} + + - name: Build ⚒️ Distributions + run: poetry build + + - name: Publish to PyPI 📦 + uses: pypa/gh-action-pypi-publish@bea5cda687c2b79989126d589ef4411bedce0195 + with: + user: __token__ + password: ${{ secrets.PYPI_TOKEN }} + skip_existing: true + + - name: Publish Release Notes 🗞 + if: env.IS_TAG_BUILD + env: + GITHUB_TAG: ${{ github.ref }} + GITHUB_REPO_SLUG: ${{ github.repository }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + GITHUB_TAG=${GITHUB_TAG/refs\/tags\//} + sudo apt-get update + sudo apt-get -y install pandoc + pip install -U github3.py pep440_version_utils + python3 ${GITHUB_WORKSPACE}/scripts/publish_gh_release_notes.py + + - name: Get RASA SDK Version + env: + RASA_SDK_VERSION: ${{ github.ref }} + run: | + echo "RASA_SDK_VERSION=${RASA_SDK_VERSION/refs\/tags\//}" >> $GITHUB_ENV + + - name: Notify Slack 💬 + if: env.IS_TAG_BUILD && success() + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_RELEASE_WEBHOOK }} + uses: Ilshidur/action-slack@2.1.0 + with: + args: "⚡ New *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} has been released! Changelog: https://github.com/RasaHQ/rasa-sdk/blob/${{ env.RASA_SDK_VERSION }}/CHANGELOG.mdx" + + - name: Notify Slack of Failure ⛔ + if: env.IS_TAG_BUILD && failure() + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_RELEASE_ASSISTANT_DEV_TRIBE_WEBHOOK }} + uses: Ilshidur/action-slack@2.1.0 + with: + args: "⛔️ *Rasa SDK* version ${{ env.RASA_SDK_VERSION }} could not be released 😱 GitHub Actions: https://github.com/RasaHQ/rasa-sdk/actions?query=branch%3A${{ env.RASA_SDK_VERSION }}" diff --git a/Dockerfile b/Dockerfile index e80736358..a9d38afe2 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM ubuntu:22.04 as base +FROM ubuntu:22.04 AS base # hadolint ignore=DL3005,DL3008 RUN apt-get update -qq \ @@ -16,7 +16,7 @@ RUN apt-get update -qq \ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 100 \ && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 100 -FROM base as python_builder +FROM base AS python_builder # hadolint ignore=DL3008 RUN apt-get update -qq \ @@ -26,10 +26,10 @@ RUN apt-get update -qq \ # install poetry # keep this in sync with the version in pyproject.toml and Dockerfile -ENV POETRY_VERSION 1.8.2 +ENV POETRY_VERSION=1.8.2 SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN curl -sSL https://install.python-poetry.org | python -ENV PATH "/root/.local/bin:/opt/venv/bin:${PATH}" +ENV PATH="/root/.local/bin:/opt/venv/bin:${PATH}" # install dependencies COPY . /app/ diff --git a/Makefile b/Makefile index aca74196d..57daad49c 100644 --- a/Makefile +++ b/Makefile @@ -2,29 +2,18 @@ TEST_PATH=./ -help: - @echo " clean" - @echo " Remove python artifacts and build artifacts." - @echo " lint" - @echo " Lint with ruff." - @echo " lint-docstrings" - @echo " Check docstring conventions in changed files." - @echo " test" - @echo " Run py.test" - @echo " init" - @echo " Install Rasa SDK dependencies" - @echo " release" - @echo " Prepare a new release" - -install: +help: ## show help message + @grep -E '^[a-z.A-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' + +install: ## install dependencies poetry run python -m pip install -U pip poetry install -install-dev: +install-dev: ## install dependencies for development poetry run python -m pip install -U pip poetry install --with dev -clean: +clean: ## remove all build, test, coverage and Python artifacts find . -name '*.pyc' -exec rm -f {} + find . -name '*.pyo' -exec rm -f {} + find . -name '*~' -exec rm -f {} + @@ -32,13 +21,13 @@ clean: rm -rf dist/ rm -rf *.egg-info -types: +types: ## check types poetry run mypy rasa_sdk -formatter: +formatter: ## format code poetry run black rasa_sdk tests -lint: +lint: ## check style with ruff and black poetry run ruff check rasa_sdk tests --ignore D poetry run black --exclude="rasa_sdk/grpc_py" --check rasa_sdk tests make lint-docstrings @@ -46,16 +35,36 @@ lint: # Compare against `main` if no branch was provided BRANCH ?= main -lint-docstrings: +lint-docstrings: ## check docstrings ./scripts/lint_python_docstrings.sh $(BRANCH) -test: clean +IMAGE_NAME ?= rasa/rasa-sdk +IMAGE_TAG ?= latest +PLATFORM ?= linux/arm64 + +build-docker: ## build docker image for one platform + docker build . \ + --platform=$(PLATFORM) \ + -f Dockerfile \ + -t $(IMAGE_NAME):$(IMAGE_TAG) + +# To be able to build a multiplatform docker image +# make sure that builder with appropriate docker driver is enabled and set as current +build-and-push-multi-platform-docker: PLATFORM = linux/amd64,linux/arm64 +build-and-push-multi-platform-docker: ## build and push multi-platform docker image + docker buildx build . \ + --platform=$(PLATFORM) \ + -f Dockerfile \ + -t $(IMAGE_NAME):$(IMAGE_TAG) \ + --push + +test: clean ## run tests poetry run py.test tests --cov rasa_sdk -v -generate-pending-changelog: +generate-pending-changelog: ## generate a changelog for the next release poetry run python -c "from scripts import release; release.generate_changelog('major.minor.patch')" -cleanup-generated-changelog: +cleanup-generated-changelog: ## cleanup the generated changelog # this is a helper to cleanup your git status locally after running "make test-docs" # it's not run on CI at the moment git status --porcelain | sed -n '/^D */s///p' | xargs git reset HEAD @@ -63,10 +72,10 @@ cleanup-generated-changelog: git ls-files --deleted | xargs git checkout git checkout CHANGELOG.mdx -release: +release: ## start the release process poetry run python scripts/release.py -generate-grpc: +generate-grpc: ## generate grpc code poetry run python -m grpc_tools.protoc \ -Irasa_sdk/grpc_py=./proto \ --python_out=. \ @@ -76,6 +85,7 @@ generate-grpc: proto/health.proto check-generate-grpc-code-in-sync: generate-grpc +check-generate-grpc-code-in-sync: ## check if the generated code is in sync with the proto files # this is a helper to check if the generated code is in sync with the proto files # it's not run on CI at the moment git diff --exit-code rasa_sdk/grpc_py | if [ "$$(wc -c)" -eq 0 ]; then echo "Generated code is in sync with proto files"; else echo "Generated code is not in sync with proto files"; exit 1; fi \ No newline at end of file diff --git a/docker-bake.hcl b/docker-bake.hcl deleted file mode 100644 index dfd5e97c7..000000000 --- a/docker-bake.hcl +++ /dev/null @@ -1,12 +0,0 @@ -variable "IMAGE_NAME" { - default = "rasa/rasa-sdk" -} - -variable "IMAGE_TAG" { - default = "main" -} - -target "default" { - dockerfile = "./Dockerfile" - tags = ["${IMAGE_NAME}:${IMAGE_TAG}"] -} From e3526637ebabe85d3b9a0910a27c3c138076a266 Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Wed, 10 Jul 2024 12:42:22 +0200 Subject: [PATCH 2/3] Set POETRY_VERSION as build argument in Dockerfile --- Dockerfile | 4 +++- Makefile | 5 ++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index a9d38afe2..4b3cb2e64 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,8 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 100 \ FROM base AS python_builder +ARG POETRY_VERSION=1.8.2 + # hadolint ignore=DL3008 RUN apt-get update -qq \ && apt-get install -y --no-install-recommends \ @@ -26,7 +28,7 @@ RUN apt-get update -qq \ # install poetry # keep this in sync with the version in pyproject.toml and Dockerfile -ENV POETRY_VERSION=1.8.2 +ENV POETRY_VERSION=$POETRY_VERSION SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN curl -sSL https://install.python-poetry.org | python ENV PATH="/root/.local/bin:/opt/venv/bin:${PATH}" diff --git a/Makefile b/Makefile index 57daad49c..22e970cac 100644 --- a/Makefile +++ b/Makefile @@ -41,9 +41,11 @@ lint-docstrings: ## check docstrings IMAGE_NAME ?= rasa/rasa-sdk IMAGE_TAG ?= latest PLATFORM ?= linux/arm64 +POETRY_VERSION ?= $(shell ./scripts/poetry-version.sh) build-docker: ## build docker image for one platform docker build . \ + --build-arg POETRY_VERSION=$(POETRY_VERSION) \ --platform=$(PLATFORM) \ -f Dockerfile \ -t $(IMAGE_NAME):$(IMAGE_TAG) @@ -53,7 +55,8 @@ build-docker: ## build docker image for one platform build-and-push-multi-platform-docker: PLATFORM = linux/amd64,linux/arm64 build-and-push-multi-platform-docker: ## build and push multi-platform docker image docker buildx build . \ - --platform=$(PLATFORM) \ + --build-arg POETRY_VERSION=$(POETRY_VERSION) \ + --platform=$(PLATFORM) \ -f Dockerfile \ -t $(IMAGE_NAME):$(IMAGE_TAG) \ --push From 158a083bd3857d5d237f4a50a38c4485929a4d91 Mon Sep 17 00:00:00 2001 From: Radovan Zivkovic Date: Wed, 10 Jul 2024 16:15:57 +0200 Subject: [PATCH 3/3] Add workflow_dispatch to release-artifacts GitHub workflow to allow manual release of Docker images and PyPI packages --- .github/workflows/release-artifacts.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/release-artifacts.yml b/.github/workflows/release-artifacts.yml index d31ff31ac..956e1fa42 100644 --- a/.github/workflows/release-artifacts.yml +++ b/.github/workflows/release-artifacts.yml @@ -4,6 +4,7 @@ on: push: tags: - "*" + workflow_dispatch: jobs: build_docker_image_set_env: