Add integration tests for gRPC

.dockerignore

@@ -8,4 +8,4 @@ CHANGELOG.txt
 .pre-commit-config.yaml
 changelog/*
 gha-creds-*.json
-**/gha-creds-*.json
+**/gha-creds-*.json

.github/actions/auth-aws-ecr/action.yml

@@ -0,0 +1,30 @@
+name: Authenticate to AWS ECR
+description: Encapsulates steps for Authenticating to ECR
+
+permissions:
+  id-token: write
+  contents: read
+
+inputs:
+  AWS_REGION:
+    description: 'AWS Region'
+    required: true
+  AWS_ARN_ROLE_TO_ASSUME:
+    description: 'AWS role ARN'
+    required: true
+
+runs:
+  using: 'composite'
+  steps:
+      - name: Configure AWS credentials
+        uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1
+        with:
+          role-to-assume: ${{ inputs.AWS_ARN_ROLE_TO_ASSUME }}
+          aws-region: ${{ inputs.AWS_REGION }}
+
+      - name: Login to Amazon ECR
+        id: login-ecr
+        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
+        with:
+          mask-password: "true"
+

.github/actions/debug-grpc/action.yml

@@ -0,0 +1,39 @@
+name: Debug custom actions integration test containers
+description: Encapsulates steps for custom actions test debugging
+
+inputs:
+  COMPOSE_FILE_PATH:
+    description: 'Custom action docker compose path'
+    required: true
+  RASA_SDK_REPOSITORY:
+    description: 'Rasa SDK repository path'
+    required: true
+  RASA_SDK_IMAGE_TAG:
+    description: 'Rasa SDK image tag'
+    required: true
+
+runs:
+  using: 'composite'
+
+  steps:
+  - name: List containers
+    run: sudo docker ps -a
+    shell: bash
+
+  - name: Check logs for action server without TLS
+    env:
+      RASA_SDK_REPOSITORY: ${{ inputs.RASA_SDK_REPOSITORY }}
+      RASA_SDK_IMAGE_TAG: ${{ inputs.RASA_SDK_IMAGE_TAG }}
+    run: |
+      docker compose -f ${{ inputs.COMPOSE_FILE_PATH }} \
+        logs action-server-grpc-no-tls
+    shell: bash
+
+  - name: Check logs for action server with TLS
+    env:
+      RASA_SDK_REPOSITORY: ${{ inputs.RASA_SDK_REPOSITORY }}
+      RASA_SDK_IMAGE_TAG: ${{ inputs.RASA_SDK_IMAGE_TAG }}
+    run: |
+      docker compose -f ${{ inputs.COMPOSE_FILE_PATH }} \
+      logs action-server-grpc-tls
+    shell: bash

.github/workflows/continuous-integration.yml

@@ -12,6 +12,10 @@ env:
   IS_TAG_BUILD: ${{ startsWith(github.event.ref, 'refs/tags') }}
   DEV_REPOSITORY: 329710836760.dkr.ecr.us-east-1.amazonaws.com/rasa-sdk-dev
   AWS_REGION: us-east-1
+  # This tag is used to build the image without dev dependencies
+  DEV_IMAGE_TAG: pr${{ github.event.number }}
+  # This tag is used to build the image with dev dependencies
+  DEV_IMAGE_WITH_DEV_DEPS_TAG: pr${{ github.event.number }}-with-dev-deps
 
 # SECRETS
 # - PYPI_TOKEN: publishing token for amn41 account, needs to be maintainer of
@@ -143,17 +147,37 @@ jobs:
       - name: Check out code
         uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
 
-      - name: Configure AWS credentials
-        uses: aws-actions/configure-aws-credentials@8c3f20df09ac63af7b3ae3d7c91f105f857d8497 # v3.0.1
+      - name: Authenticate to AWS ECR
+        uses: ./.github/actions/auth-aws-ecr
         with:
-          role-to-assume: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
-          aws-region: ${{ env.AWS_REGION }}
+          AWS_REGION: ${{ env.AWS_REGION }}
+          AWS_ARN_ROLE_TO_ASSUME: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
 
-      - name: Login to Amazon ECR
-        id: login-ecr
-        uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@0d103c3126aa41d772a8362f6aa67afac040f80c  # v3.1.0
+
+      - name: Build and push docker image to AWS
+        run: |
+          IMAGE_NAME=${{ env.DEV_REPOSITORY }} \
+          IMAGE_TAG=${{ env.DEV_IMAGE_TAG }} \
+          make build-and-push-multi-platform-docker
+
+  rasa-sdk-with-dev-deps-docker-image:
+    name: Build dev Docker image with dev dependencies
+    runs-on: ubuntu-22.04
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Authenticate to AWS ECR
+        uses: ./.github/actions/auth-aws-ecr
         with:
-          mask-password: "true"
+          AWS_REGION: ${{ env.AWS_REGION }}
+          AWS_ARN_ROLE_TO_ASSUME: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
@@ -163,6 +187,79 @@ jobs:
 
       - name: Build and push docker image to AWS
         run: |
-          IMAGE_NAME=${{ env.DEV_REPOSITORY }} \
-          IMAGE_TAG=pr${{ github.event.number }} \
-          make build-and-push-multi-platform-docker
+          IMAGE_WITH_DEV_DEPS=${{ env.DEV_REPOSITORY }} \
+          IMAGE_TAG=${{ env.DEV_IMAGE_WITH_DEV_DEPS_TAG }} \
+          make build-and-push-multi-platform-docker-with-dev-deps
+
+  grpc_standalone_integration_tests:
+    name: Run gRPC integration tests using standalone server
+    runs-on: ubuntu-22.04
+    needs: [rasa-sdk-with-dev-deps-docker-image]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Authenticate to AWS ECR
+        uses: ./.github/actions/auth-aws-ecr
+        with:
+          AWS_REGION: ${{ env.AWS_REGION }}
+          AWS_ARN_ROLE_TO_ASSUME: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
+
+      - name: Docker version
+        run: docker --version && docker compose version
+
+      - name: gRPC Server Integration Testing - Run tests on gRPC server with and without TLS 🩺
+        run: |
+          make run-grpc-standalone-integration-tests
+        env:
+          IMAGE_WITH_DEV_DEPS: ${{ env.DEV_REPOSITORY }}
+          IMAGE_TAG: ${{ env.DEV_IMAGE_WITH_DEV_DEPS_TAG }}
+
+
+  grpc_docker_integration_tests:
+    name: Run gRPC integration tests using Docker containers
+    runs-on: ubuntu-22.04
+    needs: [rasa-sdk-dev-docker-image, rasa-sdk-with-dev-deps-docker-image]
+
+    steps:
+      - name: Check out code
+        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
+
+      - name: Authenticate to AWS ECR
+        uses: ./.github/actions/auth-aws-ecr
+        with:
+          AWS_REGION: ${{ env.AWS_REGION }}
+          AWS_ARN_ROLE_TO_ASSUME: ${{ secrets.AWS_ASSUME_ROLE_SESSION_TOKEN }}
+
+      - name: Docker version
+        run: docker --version && docker compose version
+
+      - name: gRPC Server Integration Testing - Run env docker containers
+        run: |
+          make start-grpc-integration-test-env
+        env:
+          RASA_SDK_REPOSITORY: ${{ env.DEV_REPOSITORY }}
+          RASA_SDK_IMAGE_TAG: ${{ env.DEV_IMAGE_TAG }}
+
+      - name: gRPC Server Integration Testing - Run tests on gRPC server with and without TLS 🩺
+        run: |
+          make run-grpc-integration-tests
+        env:
+          IMAGE_WITH_DEV_DEPS: ${{ env.DEV_REPOSITORY }}
+          IMAGE_TAG: ${{ env.DEV_IMAGE_WITH_DEV_DEPS_TAG }}
+
+      - name: gRPC Server Integration Testing - Stop env docker containers
+        run: |
+          make stop-grpc-integration-test-env
+        env:
+          RASA_SDK_REPOSITORY: ${{ env.DEV_REPOSITORY }}
+          RASA_SDK_IMAGE_TAG: ${{ env.DEV_IMAGE_TAG }}
+
+      - name: Show container logs
+        if: always()
+        uses: ./.github/actions/debug-grpc
+        with:
+          COMPOSE_FILE_PATH: integration_tests/grpc_server/setup/docker-compose.yml
+          RASA_SDK_REPOSITORY: ${{ env.DEV_REPOSITORY }}
+          RASA_SDK_IMAGE_TAG: ${{ env.DEV_IMAGE_TAG }}

.gitignore

@@ -116,3 +116,6 @@ examples/moodbot/models/
 *.DS_Store
 tests/executor_test_packages
 .pytype/
+
+grpc-standalone-server-integration-test-results.xml
+grpc-server-docker-integration-test-results.xml

Dockerfile.dev

@@ -0,0 +1,62 @@
+FROM ubuntu:22.04 AS base
+
+# hadolint ignore=DL3005,DL3008
+RUN apt-get update -qq \
+    # Make sure that all security updates are installed
+    && apt-get dist-upgrade -y --no-install-recommends \
+    && apt-get install -y --no-install-recommends \
+      python3 \
+      python3-venv \
+      python3-pip \
+      python3-dev \
+    && apt-get autoremove -y \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 100 \
+   && update-alternatives --install /usr/bin/pip pip /usr/bin/pip3 100
+
+FROM base AS python_builder
+
+ARG POETRY_VERSION=1.8.2
+
+# hadolint ignore=DL3008
+RUN apt-get update -qq \
+   && apt-get install -y --no-install-recommends \
+    curl \
+    && apt-get autoremove -y
+
+# install poetry
+# keep this in sync with the version in pyproject.toml and Dockerfile
+ENV POETRY_VERSION=$POETRY_VERSION
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONIOENCODING="utf-8"
+
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+RUN curl -sSL https://install.python-poetry.org | python
+ENV PATH="/root/.local/bin:/opt/venv/bin:${PATH}"
+
+# install dependencies
+COPY . /app/
+
+WORKDIR /app
+
+# hadolint ignore=SC1091,DL3013
+# install dependencies and build wheels
+RUN python -m venv /opt/venv && \
+  . /opt/venv/bin/activate && \
+  pip install --no-cache-dir -U pip && \
+  pip install --no-cache-dir wheel && \
+  poetry install --no-root --no-interaction
+
+# build the Rasa SDK wheel and install it
+# hadolint ignore=SC1091,DL3013
+RUN poetry build -f wheel -n && \
+  pip install --no-deps dist/*.whl && \
+  rm -rf dist *.egg-info
+
+RUN rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* \
+  && rm -rf /root/.cache/pip/*
+
+EXPOSE 5055
+ENTRYPOINT [""]