feat: Setup e2e tests to be running in parallel by using Docker

Additionally:
– add README for e2e tests
– update env variables template
– cleanup npm scripts

.env.template

@@ -4,10 +4,17 @@ API_LAYER_API_KEY=iVH7l3yBziOKwGSO7jYWYt1RDtb05oKf
 APPLICATION_HOST=127.0.0.1
 APPLICATION_PORT=8081
 APPLICATION_JWT_SECRET=development
+# for .env.test use docker/test/docker-compose db service name
 APPLICATION_DB_HOST=127.0.0.1
 APPLICATION_DB_USERNAME=
 APPLICATION_DB_PASSWORD=
 APPLICATION_DB_DATABASE=budget-tracker
 APPLICATION_DB_PORT=5432
 APPLICATION_DB_DIALECT=postgres
+# for .env.test use docker/test/docker-compose redis service name
 APPLICATION_REDIS_HOST=127.0.0.1
+
+# Tests configurations
+# e2e tests are running in parallel, so we need a strict amount of workers,
+# so then we can  dynamically create the same amount of DBs
+JEST_WORKERS_AMOUNT=4

docker/dev/docker-destroy.sh

@@ -3,4 +3,4 @@
 echo "Starting removing all dev container completely..."
 
 npm run docker:dev -- -d
-npm run docker:dev:down -- --rmi all --volumes --remove-orphans
+npm run docker:dev:down -- --rmi all --volumes

docker/test/Dockerfile

@@ -0,0 +1,19 @@
+FROM node:21.7.3
+
+WORKDIR /app
+
+# Copy package.json and package-lock.json files. This allows Docker to cache the
+# npm dependencies as long as these files don't change.
+COPY package*.json ./
+
+# Install dependencies
+COPY post-install.sh ./
+COPY docker ./docker
+RUN chmod +x ./post-install.sh
+RUN npm ci
+
+# Copy the rest of the application
+COPY . .
+
+# Run this command to keep container alive. Without it will be demounted right after deps installation
+CMD ["tail", "-f", "/dev/null"]

docker/test/docker-compose.yml

@@ -0,0 +1,33 @@
+services:
+  test-db:
+    image: postgres:16
+    restart: always
+    container_name: test-budget-tracker-db
+    volumes: ['test_db_data:/var/lib/postgresql/data']
+    environment:
+      - POSTGRES_USER=${APPLICATION_DB_USERNAME}
+      - POSTGRES_PASSWORD=${APPLICATION_DB_PASSWORD}
+      - POSTGRES_DB=${APPLICATION_DB_DATABASE}
+    ports: ['${APPLICATION_DB_PORT}:5432']
+    env_file: ../../.env.test
+
+  test-redis:
+    image: redis:6
+    container_name: test-budget-tracker-redis
+    volumes: ['test_redis_data:/data']
+    ports: ['6379:6379']
+
+  test-runner:
+    build:
+      context: ../..
+      dockerfile: docker/test/Dockerfile
+    depends_on:
+      - test-db
+      - test-redis
+    environment:
+      - NODE_ENV=test
+    env_file: ../../.env.test
+
+volumes:
+  test_db_data:
+  test_redis_data:

jest.config.e2e.ts

@@ -5,6 +5,7 @@ console.log('❗ RUNNING INTEGRATION TESTS');
 /** @type {import('ts-jest/dist/types').InitialOptionsTsJest} */
 export default {
   ...baseConfig,
+  maxWorkers: process.env.JEST_WORKERS_AMOUNT,
   testMatch: ['<rootDir>/src/**/?(*.)+(e2e).[jt]s?(x)'],
   setupFilesAfterEnv: ['<rootDir>/src/tests/setupIntegrationTests.ts'],
 };

package.json

@@ -10,11 +10,9 @@
     "migrate:dev:undo": "cross-env NODE_ENV=development npx sequelize-cli db:migrate:undo",
     "migrate": "cross-env NODE_ENV=production npx sequelize-cli db:migrate",
     "migrate:undo": "cross-env NODE_ENV=production npx sequelize-cli db:migrate:undo",
-    "db:reset": "cross-env NODE_ENV=test npx sequelize-cli db:drop && npx sequelize-cli db:create && npx sequelize-cli db:migrate",
-    "pretest": "cross-env NODE_ENV=test npm run db:reset",
     "test": "cross-env NODE_ENV=test npm run test:unit && npm run test:e2e",
     "test:unit": "cross-env NODE_ENV=test jest -c jest.config.unit.ts --passWithNoTests --forceExit --detectOpenHandles",
-    "test:e2e": "cross-env NODE_ENV=test jest -c jest.config.e2e.ts --runInBand --passWithNoTests --forceExit --detectOpenHandles",
+    "test:e2e": "chmod +x ./src/tests/setup-e2e-tests.sh && ./src/tests/setup-e2e-tests.sh",
     "lint": "eslint .",
     "docker:dev": "docker compose --env-file .env.development -f ./docker/dev/docker-compose.yml up --build",
     "docker:dev:ps": "docker compose --env-file .env.development -f ./docker/dev/docker-compose.yml ps",

src/app.ts

@@ -34,6 +34,8 @@ export const app = express();
 const apiPrefix = config.get('apiPrefix');
 export const redisClient = createClient({
   host: config.get('redis.host'),
+  keyPrefix:
+    process.env.NODE_ENV === 'test' ? `test-worker-${process.env.JEST_WORKER_ID}` : undefined,
 });
 
 redisClient.on('error', (error: Error) => {

src/models/index.ts

@@ -16,6 +16,10 @@ const DBConfig: Record<string, unknown> = config.get('db');
 
 const sequelize = new Sequelize({
   ...DBConfig,
+  database:
+    process.env.NODE_ENV === 'test'
+      ? `${DBConfig.database}-${process.env.JEST_WORKER_ID}`
+      : (DBConfig.database as string),
   models: [__dirname + '/**/*.model.ts'],
   pool: {
     max: 50,

src/tests/README.md

@@ -0,0 +1,47 @@
+## End-to-End (E2E) Tests Setup
+
+The e2e tests setup is designed to efficiently run tests in parallel using multiple databases. Below is a detailed description of the setup process and necessary configurations.
+
+### Overview
+
+The current implementation of E2E tests uses multiple databases to facilitate parallel test execution. Since each test expects that it will work with the fresh empty DB, we need to empty it before each test suite. Without multiple DBs, it means we can run tests only sequentially.
+
+### Jest Configuration
+
+We use Jest as our testing framework and have defined `JEST_WORKERS_AMOUNT` workers to run the tests in parallel. Each worker requires a separate database instance.
+
+### Database Setup
+
+For each Jest worker, a corresponding database is created with the naming convention `{APPLICATION_DB_DATABASE}-{n}`, where `n` is the worker ID. This worker ID ranges exactly as `{1...JEST_WORKERS_AMOUNT}`.
+
+### Database Connection
+
+The database connection is specified in the src/models/index.ts file. Here, we dynamically assign the database name based on the Jest worker ID.
+
+```ts
+database: process.env.NODE_ENV === 'test'
+  ? `${DBConfig.database}-${process.env.JEST_WORKER_ID}`
+  : (DBConfig.database as string),
+```
+
+### Redis Connection
+
+The Redis connection is also dynamically specified per each test worker by setting `keyPrefix` to a custom value like below in `src/app.ts`:
+
+```ts
+export const redisClient = createClient({
+  host: config.get('redis.host'),
+  keyPrefix:
+    process.env.NODE_ENV === 'test' ? `test-worker-${process.env.JEST_WORKER_ID}` : undefined,
+});
+```
+
+It's important to set the prefix so that parallel tests won't conflict with the same Redis service. It doesn't affect developer experience since when you work with Redis by keys, the prefix is assigned automatically, `keyPrefix` doesn't affect development at all.
+
+### Docker Integration
+
+To simplify the setup and avoid conflicts with the local environment, we use Docker to manage our databases and the application.
+
+The application is also containerized. We run our tests from within this Docker container to ensure it can communicate with the database containers.
+
+All Docker configs for tests are stored under `./docker/test/` directory.

src/tests/setup-e2e-tests.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+
+# Source environment variables from .env.test file
+if [ -f .env.test ]; then
+    export $(cat .env.test | grep -v '#' | awk '/=/ {print $1}')
+else
+    echo ".env.test file not found"
+    exit 1
+fi
+
+# Start the containers and run tests
+docker compose -f ./docker/test/docker-compose.yml up --build -d
+
+echo "Waiting a bit..."
+sleep 3
+
+echo "Creating databases..."
+
+docker compose -f ./docker/test/docker-compose.yml exec -T test-db bash -c "
+for i in \$(seq 1 \$JEST_WORKERS_AMOUNT); do
+  psql -U \"${APPLICATION_DB_USERNAME}\" -d postgres -c \"DROP DATABASE IF EXISTS \\\"${APPLICATION_DB_DATABASE}-\$i\\\";\"
+  psql -U \"${APPLICATION_DB_USERNAME}\" -d postgres -c \"CREATE DATABASE \\\"${APPLICATION_DB_DATABASE}-\$i\\\";\"
+done
+"
+
+echo "Running tests..."
+# Run tests
+docker compose -f ./docker/test/docker-compose.yml exec -T test-runner \
+  npx jest -c jest.config.e2e.ts --passWithNoTests --forceExit --colors "$@"
+
+# Capture the exit code
+TEST_EXIT_CODE=$?
+
+# Clean up
+docker compose -f ./docker/test/docker-compose.yml down -v
+
+# Check the exit code and display an error message if it's 1
+if [ $TEST_EXIT_CODE -eq 1 ]; then
+    echo -e "\n\n$(tput setaf 1)ERROR: Tests failed!$(tput sgr0)"
+else
+    echo -e "\n\n$(tput setaf 2)Tests passed successfully.$(tput sgr0)"
+fi
+
+# Exit with the test exit code
+exit $TEST_EXIT_CODE