docker-compose.yml

services:

  postgres:
    image: postgres:latest
    container_name: postgres
    ports:
      - "5432:5432"
    environment:
      - POSTGRES_USER=${POSTGRES_USER}
      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD}
      - POSTGRES_DB=northwind
    volumes:
      - ./docker/postgres/databases:/docker-entrypoint-initdb.d
      - ./docker/postgres/init-db.sh:/docker-entrypoint-initdb.d/init-db.sh
    command: ["postgres", "-c", "log_statement=all"]
    networks:
      - dqsdatalabs

  
  minio:
    image: minio/minio:latest
    container_name: minio
    ports:
      - "9000:9000"
      - "9001:9001"      
    environment:
      - MINIO_ROOT_USER=${MINIO_ROOT_USER}
      - MINIO_ROOT_PASSWORD=${MINIO_ROOT_PASSWORD}
      - MINIO_DOMAIN=minio
    volumes:
      - minio-data:/data
    networks:
      - dqsdatalabs
    command: ["server", "/data", "--console-address", ":9001"]

  minio-mc:
    image: minio/mc:latest
    container_name: minio-mc
    environment:
      - MINIO_ACCESS_KEY=minio
      - MINIO_SECRET_KEY=minio123
    volumes:
      - ./docker/minio/minio-entrypoint.sh:/usr/bin/entrypoint.sh
    entrypoint: /bin/sh /usr/bin/entrypoint.sh
    depends_on:
      - minio
    networks:
      - dqsdatalabs
  spark-master:
    build: 
      context: ./docker/flink
      dockerfile: Dockerfile
    image: spark-sandbox:latest
    container_name: spark-master
    environment:
      - SPARK_MASTER_HOST=spark-master
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 8080:8080
    command: /opt/spark/bin/spark-class org.apache.spark.deploy.master.Master
    networks:
      - dqsdatalabs

  spark-worker:
    build: 
      context: ./docker/spark
      dockerfile: Dockerfile
    image: spark-sandbox:latest
    container_name: spark-worker-1
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_WEBUI_PORT=8082
    depends_on:
      - spark-master
    ports:
      - "8082:8082"
    command: /opt/spark/bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077
    networks:
      - dqsdatalabs
    deploy:
        resources:
          limits:
            cpus: "3.0"
            memory: 2G

  flink-jobmanager:
    build: 
      context: ./docker/flink
      dockerfile: Dockerfile
    image: flink-sandbox:latest
    container_name: flink-jobmanager
    depends_on:
      - minio
    ports:
      - "8081:8081"
    expose:
      - "6123"
    environment:
      FLINK_MODE: jobmanager
      FLINK_CFG_REST_PORT: 8081
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
    command: jobmanager
    networks:
      - dqsdatalabs

  flink-taskmanager:
    build: 
      context: ./docker/flink
      dockerfile: Dockerfile
    image: flink-sandbox:latest
    container_name: flink-taskmanager
    depends_on:
      - flink-jobmanager
    environment:
      FLINK_MODE: taskmanager
      FLINK_PROPERTIES: |
        jobmanager.rpc.address: flink-jobmanager
    command: taskmanager
    networks:
      - dqsdatalabs

  jupyter:
    build: 
      context: ./docker/jupyter
      dockerfile: Dockerfile
    image: jupyter-sandbox:latest
    container_name: jupyter-notebooks
    ports:
      - "8888:8888"
    environment:
      - JUPYTER_ENABLE_LAB=yes
    volumes:
      - ./docker/jupyter/datasets:/home/jovyan/datasets
      - ./docker/jupyter/notebooks:/home/jovyan/notebooks
    networks:
      - dqsdatalabs

networks:
  dqsdatalabs:
    name: dqsdatalabs
    driver: bridge
    ipam:
      config:
        - subnet: "172.50.0.0/24"
          gateway: "172.50.0.1"

volumes:
  minio-data: