docker-compose.yaml

version: "3.8"
#https://github.com/compose-spec/compose-spec/blob/master/spec.md#using-extensions-as-fragments
x-logging: &default-logging
  options:
    max-size: "100m"
    max-file: "5"
  driver: json-file
services:
  docker-proxy:
      image: alpine/socat
      command: -t 900 TCP-LISTEN:2375,fork,reuseaddr UNIX-CONNECT:/var/run/docker.sock
      ports: 
        - "2375"
      user: root
      volumes: 
        - ${DOCKER_SOCKET:-/var/run/docker.sock}:/var/run/docker.sock
      networks:
        - airbyte_internal
  # hook in case we need to add init behavior
  # every root service (no depends_on) should depend on init
  init:
    image: airbyte/init:${VERSION}
    logging: *default-logging
    container_name: init
    command: /bin/sh -c "./scripts/create_mount_directories.sh /local_parent ${HACK_LOCAL_ROOT_PARENT} ${LOCAL_ROOT}"
    environment:
      - LOCAL_ROOT=${LOCAL_ROOT}
      - HACK_LOCAL_ROOT_PARENT=${HACK_LOCAL_ROOT_PARENT}
    volumes:
      - ${HACK_LOCAL_ROOT_PARENT}:/local_parent
  bootloader:
    image: airbyte/bootloader:${VERSION}
    logging: *default-logging
    container_name: airbyte-bootloader
    environment:
      - AIRBYTE_VERSION=${VERSION}
      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
      - CONNECTOR_REGISTRY_SEED_PROVIDER=${CONNECTOR_REGISTRY_SEED_PROVIDER:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - LOCAL_CONNECTOR_CATALOG_PATH=${LOCAL_CONNECTOR_CATALOG_PATH}
      - LOG_LEVEL=${LOG_LEVEL}
    networks:
      - airbyte_internal
    depends_on:
      init:
        condition: service_completed_successfully
  db:
    image: airbyte/db:${VERSION}
    logging: *default-logging
    container_name: airbyte-db
    restart: unless-stopped
    environment:
      - CONFIG_DATABASE_PASSWORD=${CONFIG_DATABASE_PASSWORD:-}
      - CONFIG_DATABASE_URL=${CONFIG_DATABASE_URL:-}
      - CONFIG_DATABASE_USER=${CONFIG_DATABASE_USER:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - POSTGRES_PASSWORD=${DATABASE_PASSWORD}
      - POSTGRES_USER=${DATABASE_USER}
    volumes:
      - db:/var/lib/postgresql/data
    networks:
      - airbyte_internal
  worker:
    image: airbyte/worker:${VERSION}
    logging: *default-logging
    container_name: airbyte-worker
    restart: unless-stopped
    environment:
      - ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS=${ACTIVITY_INITIAL_DELAY_BETWEEN_ATTEMPTS_SECONDS}
      - ACTIVITY_MAX_ATTEMPT=${ACTIVITY_MAX_ATTEMPT}
      - ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS=${ACTIVITY_MAX_DELAY_BETWEEN_ATTEMPTS_SECONDS}
      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
      - AIRBYTE_VERSION=${VERSION}
      - APPLY_FIELD_SELECTION=${APPLY_FIELD_SELECTION}
      - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA}
      - AUTO_DISABLE_FAILING_CONNECTIONS=${AUTO_DISABLE_FAILING_CONNECTIONS}
      - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-}
      - CONFIG_DATABASE_PASSWORD=${CONFIG_DATABASE_PASSWORD:-}
      - CONFIG_DATABASE_URL=${CONFIG_DATABASE_URL:-}
      - CONFIG_DATABASE_USER=${CONFIG_DATABASE_USER:-}
      - CONFIG_ROOT=${CONFIG_ROOT}
      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
      - DD_AGENT_HOST=${DD_AGENT_HOST}
      - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT}
      - DOCKER_HOST=docker-proxy:2375
      - FEATURE_FLAG_CLIENT=${FEATURE_FLAG_CLIENT}
      - FIELD_SELECTION_WORKSPACES=${FIELD_SELECTION_WORKSPACES}
      - INTERNAL_API_HOST=${INTERNAL_API_HOST}
      - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-}
      - JOB_ERROR_REPORTING_SENTRY_DSN=${JOB_ERROR_REPORTING_SENTRY_DSN}
      - JOB_ERROR_REPORTING_STRATEGY=${JOB_ERROR_REPORTING_STRATEGY}
      - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT}
      - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST}
      - JOB_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_MAIN_CONTAINER_MEMORY_LIMIT}
      - JOB_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_MAIN_CONTAINER_MEMORY_REQUEST}
      - LAUNCHDARKLY_KEY=${LAUNCHDARKLY_KEY}
      - LOCAL_DOCKER_MOUNT=${LOCAL_DOCKER_MOUNT}
      - LOCAL_ROOT=${LOCAL_ROOT}
      - LOG_CONNECTOR_MESSAGES=${LOG_CONNECTOR_MESSAGES}
      - LOG_LEVEL=${LOG_LEVEL}
      - MAX_CHECK_WORKERS=${MAX_CHECK_WORKERS}
      - MAX_DISCOVER_WORKERS=${MAX_DISCOVER_WORKERS}
      - MAX_NOTIFY_WORKERS=${MAX_NOTIFY_WORKERS}
      - MAX_SPEC_WORKERS=${MAX_SPEC_WORKERS}
      - MAX_SYNC_WORKERS=${MAX_SYNC_WORKERS}
      - METRIC_CLIENT=${METRIC_CLIENT}
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - MICRONAUT_ENVIRONMENTS=${WORKERS_MICRONAUT_ENVIRONMENTS}
      - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_LIMIT}
      - NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_CPU_REQUEST}
      - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_LIMIT}
      - NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST=${NORMALIZATION_JOB_MAIN_CONTAINER_MEMORY_REQUEST}
      - OTEL_COLLECTOR_ENDPOINT=${OTEL_COLLECTOR_ENDPOINT}
      - PUBLISH_METRICS=${PUBLISH_METRICS}
      - SECRET_PERSISTENCE=${SECRET_PERSISTENCE}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - SHOULD_RUN_NOTIFY_WORKFLOWS=${SHOULD_RUN_NOTIFY_WORKFLOWS}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - SYNC_JOB_INIT_RETRY_TIMEOUT_MINUTES=${SYNC_JOB_INIT_RETRY_TIMEOUT_MINUTES}
      - SYNC_JOB_MAX_ATTEMPTS=${SYNC_JOB_MAX_ATTEMPTS}
      - SYNC_JOB_MAX_TIMEOUT_DAYS=${SYNC_JOB_MAX_TIMEOUT_DAYS}
      - TEMPORAL_HOST=${TEMPORAL_HOST}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
      - WEBAPP_URL=${WEBAPP_URL}
      - STORAGE_BUCKET_ACTIVITY_PAYLOAD=${STORAGE_BUCKET_ACTIVITY_PAYLOAD}
      - STORAGE_BUCKET_LOG=${STORAGE_BUCKET_LOG}
      - STORAGE_BUCKET_STATE=${STORAGE_BUCKET_STATE}
      - STORAGE_BUCKET_WORKLOAD_OUTPUT=${STORAGE_BUCKET_WORKLOAD_OUTPUT}
      - STORAGE_TYPE=${STORAGE_TYPE}
      - WORKFLOW_FAILURE_RESTART_DELAY_SECONDS=${WORKFLOW_FAILURE_RESTART_DELAY_SECONDS}
      - WORKLOAD_API_HOST=${WORKLOAD_API_URL}
      - WORKSPACE_DOCKER_MOUNT=${WORKSPACE_DOCKER_MOUNT}
      - WORKSPACE_ROOT=${WORKSPACE_ROOT}
    configs:
      - flags
    volumes:
      - workspace:${WORKSPACE_ROOT}
      - local_root:${LOCAL_ROOT}
    ports:
      - "9000"
    networks:
      - airbyte_internal
    depends_on:
      bootloader:
        condition: service_completed_successfully
  server:
    image: airbyte/server:${VERSION}
    logging: *default-logging
    container_name: airbyte-server
    restart: unless-stopped
    environment:
      - AIRBYTE_ROLE=${AIRBYTE_ROLE:-}
      - AIRBYTE_VERSION=${VERSION}
      - AUTO_DETECT_SCHEMA=${AUTO_DETECT_SCHEMA}
      - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-}
      - CONFIG_DATABASE_PASSWORD=${CONFIG_DATABASE_PASSWORD:-}
      - CONFIG_DATABASE_URL=${CONFIG_DATABASE_URL:-}
      - CONFIG_DATABASE_USER=${CONFIG_DATABASE_USER:-}
      - CONFIG_ROOT=${CONFIG_ROOT}
      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - DD_AGENT_HOST=${DD_AGENT_HOST}
      - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT}
      - FEATURE_FLAG_CLIENT=${FEATURE_FLAG_CLIENT}
      - GITHUB_STORE_BRANCH=${GITHUB_STORE_BRANCH}
      - JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION:-}
      - JOB_ERROR_REPORTING_SENTRY_DSN=${JOB_ERROR_REPORTING_SENTRY_DSN}
      - JOB_ERROR_REPORTING_STRATEGY=${JOB_ERROR_REPORTING_STRATEGY}
      - JOB_MAIN_CONTAINER_CPU_LIMIT=${JOB_MAIN_CONTAINER_CPU_LIMIT}
      - JOB_MAIN_CONTAINER_CPU_REQUEST=${JOB_MAIN_CONTAINER_CPU_REQUEST}
      - JOB_MAIN_CONTAINER_MEMORY_LIMIT=${JOB_MAIN_CONTAINER_MEMORY_LIMIT}
      - JOB_MAIN_CONTAINER_MEMORY_REQUEST=${JOB_MAIN_CONTAINER_MEMORY_REQUEST}
      - LAUNCHDARKLY_KEY=${LAUNCHDARKLY_KEY}
      - LOG_LEVEL=${LOG_LEVEL}
      - METRIC_CLIENT=${METRIC_CLIENT}
      - MAX_NOTIFY_WORKERS=5
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - MICRONAUT_ENVIRONMENTS=${WORKERS_MICRONAUT_ENVIRONMENTS}
      - NEW_SCHEDULER=${NEW_SCHEDULER}
      - PUBLISH_METRICS=${PUBLISH_METRICS}
      - SECRET_PERSISTENCE=${SECRET_PERSISTENCE}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - SHOULD_RUN_NOTIFY_WORKFLOWS=${SHOULD_RUN_NOTIFY_WORKFLOWS}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - STORAGE_BUCKET_ACTIVITY_PAYLOAD=${STORAGE_BUCKET_ACTIVITY_PAYLOAD}
      - STORAGE_BUCKET_LOG=${STORAGE_BUCKET_LOG}
      - STORAGE_BUCKET_STATE=${STORAGE_BUCKET_STATE}
      - STORAGE_BUCKET_WORKLOAD_OUTPUT=${STORAGE_BUCKET_WORKLOAD_OUTPUT}
      - STORAGE_TYPE=${STORAGE_TYPE}
      - TEMPORAL_HOST=${TEMPORAL_HOST}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
      - WEBAPP_URL=${WEBAPP_URL}
      - WORKER_ENVIRONMENT=${WORKER_ENVIRONMENT}
      - WORKSPACE_ROOT=${WORKSPACE_ROOT}
      - CONNECTOR_BUILDER_SERVER_API_HOST=${CONNECTOR_BUILDER_SERVER_API_HOST}
    ports:
      - "8001"
    configs:
      - flags
    volumes:
      - workspace:${WORKSPACE_ROOT}
      - data:${CONFIG_ROOT}
      - local_root:${LOCAL_ROOT}
      - ./configs:/app/configs:ro
    networks:
      - airbyte_internal
    depends_on:
      bootloader:
        condition: service_completed_successfully
  webapp:
    image: airbyte/webapp:${VERSION}
    logging: *default-logging
    container_name: airbyte-webapp
    restart: unless-stopped
    environment:
      - INTERNAL_API_HOST=${INTERNAL_API_HOST}
      - CONNECTOR_BUILDER_API_HOST=${CONNECTOR_BUILDER_API_HOST}
      - KEYCLOAK_INTERNAL_HOST=localhost # placeholder to ensure the webapp's nginx config is valid
    networks:
      - airbyte_internal
    depends_on:
      bootloader:
        condition: service_completed_successfully
  airbyte-temporal:
    image: airbyte/temporal:${VERSION}
    logging: *default-logging
    container_name: airbyte-temporal
    restart: unless-stopped
    environment:
      - DB=postgresql
      - DB_PORT=${DATABASE_PORT}
      - DYNAMIC_CONFIG_FILE_PATH=config/dynamicconfig/development.yaml
      - LOG_LEVEL=${LOG_LEVEL}
      - POSTGRES_PWD=${DATABASE_PASSWORD}
      - POSTGRES_SEEDS=${DATABASE_HOST}
      - POSTGRES_USER=${DATABASE_USER}
    volumes:
      - ./temporal/dynamicconfig:/etc/temporal/config/dynamicconfig
    networks:
      - airbyte_internal
  airbyte-cron:
    image: airbyte/cron:${VERSION}
    logging: *default-logging
    container_name: airbyte-cron
    restart: unless-stopped
    environment:
      - AIRBYTE_VERSION=${VERSION}
      - CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=${CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION}
      - CONNECTOR_REGISTRY_BASE_URL=${CONNECTOR_REGISTRY_BASE_URL:-}
      - CONNECTOR_REGISTRY_SEED_PROVIDER=${CONNECTOR_REGISTRY_SEED_PROVIDER:-}
      - DATABASE_PASSWORD=${DATABASE_PASSWORD}
      - DATABASE_URL=${DATABASE_URL}
      - DATABASE_USER=${DATABASE_USER}
      - DD_AGENT_HOST=${DD_AGENT_HOST}
      - DD_DOGSTATSD_PORT=${DD_DOGSTATSD_PORT}
      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
      - LOG_LEVEL=${LOG_LEVEL}
      - METRIC_CLIENT=${METRIC_CLIENT}
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - MICRONAUT_ENVIRONMENTS=${CRON_MICRONAUT_ENVIRONMENTS}
      - PUBLISH_METRICS=${PUBLISH_METRICS}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - TEMPORAL_HISTORY_RETENTION_IN_DAYS=${TEMPORAL_HISTORY_RETENTION_IN_DAYS}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
      - UPDATE_DEFINITIONS_CRON_ENABLED=${UPDATE_DEFINITIONS_CRON_ENABLED}
      - WORKSPACE_ROOT=${WORKSPACE_ROOT}
    configs:
      - flags
    volumes:
      - workspace:${WORKSPACE_ROOT}
    networks:
      - airbyte_internal
    depends_on:
      bootloader:
        condition: service_completed_successfully
  airbyte-api-server:
    image: airbyte/airbyte-api-server:${VERSION}
    logging: *default-logging
    container_name: airbyte-api-server
    restart: unless-stopped
    ports:
      - "8006"
    environment:
      - AIRBYTE_API_HOST=${AIRBYTE_API_HOST}
      - AIRBYTE_VERSION=${VERSION}
      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
      - INTERNAL_API_HOST=${INTERNAL_API_URL} # Non-ideal, but the existing URL can't have https:// added because it's used for nginx conf
      - LOG_LEVEL=${LOG_LEVEL}
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
    networks:
      - airbyte_internal
    depends_on:
      bootloader:
        condition: service_completed_successfully
  airbyte-connector-builder-server:
    image: airbyte/connector-builder-server:${VERSION}
    logging: *default-logging
    container_name: airbyte-connector-builder-server
    restart: unless-stopped
    ports:
      - 8080
    environment:
      - AIRBYTE_VERSION=${VERSION}
      - CDK_VERSION=${CDK_VERSION}
      - DEPLOYMENT_MODE=${DEPLOYMENT_MODE}
      - METRIC_CLIENT=${METRIC_CLIENT}
      - MICROMETER_METRICS_ENABLED=${MICROMETER_METRICS_ENABLED}
      - MICROMETER_METRICS_STATSD_FLAVOR=${MICROMETER_METRICS_STATSD_FLAVOR}
      - PYTHON_VERSION=${PYTHON_VERSION}
      - SEGMENT_WRITE_KEY=${SEGMENT_WRITE_KEY}
      - STATSD_HOST=${STATSD_HOST}
      - STATSD_PORT=${STATSD_PORT}
      - TRACKING_STRATEGY=${TRACKING_STRATEGY}
    networks:
      - airbyte_internal
    depends_on:
      bootloader:
        condition: service_completed_successfully
  airbyte-proxy:
    image: airbyte/proxy:${VERSION}
    container_name: airbyte-proxy
    restart: unless-stopped
    ports:
      - "8000:8000"
      - "8001:8001"
      - "8003:8003"
      - "8006:8006"
    environment:
      - BASIC_AUTH_USERNAME=${BASIC_AUTH_USERNAME}
      - BASIC_AUTH_PASSWORD=${BASIC_AUTH_PASSWORD}
      - BASIC_AUTH_PROXY_TIMEOUT=${BASIC_AUTH_PROXY_TIMEOUT}
    networks:
      - airbyte_internal
      - airbyte_public
    depends_on:
      - webapp
      - server
      - airbyte-api-server
volumes:
  workspace:
    name: ${WORKSPACE_DOCKER_MOUNT}
  local_root:
    name: ${LOCAL_DOCKER_MOUNT}
  # the data volume is only needed for backward compatibility; when users upgrade
  # from an old Airbyte version that relies on file-based configs, the server needs
  # to read this volume to copy their configs to the database
  data:
    name: ${DATA_DOCKER_MOUNT}
  db:
    name: ${DB_DOCKER_MOUNT}
configs:
  flags:
    file: ./flags.yml
networks:
  airbyte_public:
  airbyte_internal: