docker-compose.yml

services:
  postgres_oltp:
    image: postgres:${POSTGRES_IMAGE_TAG}
    container_name: postgres_oltp
    restart: unless-stopped
    command:
      - postgres
      - -c
      - wal_level=logical
      - -c
      - max_replication_slots=16
      - -c
      - max_wal_senders=10
    environment:
      POSTGRES_USER: ${PG_OLTP_USER}
      POSTGRES_PASSWORD: ${PG_OLTP_PASSWORD}
      POSTGRES_DB: ${PG_OLTP_DB}
      DEBEZIUM_OLTP_PASSWORD: ${DEBEZIUM_OLTP_PASSWORD:-debezium_oltp_pass}
    ports:
      - "${PG_OLTP_PORT}:5432"
    volumes:
      - pg_oltp_data:/var/lib/postgresql/data
      - ./services/postgres/init/02_oltp_schema.sql:/docker-entrypoint-initdb.d/02_oltp_schema.sql:ro
      - ./services/postgres/init/02b_oltp_marketing_hr_finance.sql:/docker-entrypoint-initdb.d/02b_oltp_marketing_hr_finance.sql:ro
      - ./services/postgres/init/02c_oltp_retail_legacy.sql:/docker-entrypoint-initdb.d/02c_oltp_retail_legacy.sql:ro
      - ./services/postgres/cdc/oltp_debezium_bootstrap.sh:/docker-entrypoint-initdb.d/99_debezium_bootstrap.sh:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${PG_OLTP_USER} -d ${PG_OLTP_DB}"]
      interval: 10s
      timeout: 5s
      retries: 10
    networks:
      - dataops_net

  postgres_olap:
    image: postgres:${POSTGRES_IMAGE_TAG}
    container_name: postgres_olap
    restart: unless-stopped
    command:
      - postgres
      - -c
      - wal_level=logical
      - -c
      - max_replication_slots=16
      - -c
      - max_wal_senders=10
    environment:
      POSTGRES_USER: ${PG_OLAP_USER}
      POSTGRES_PASSWORD: ${PG_OLAP_PASSWORD}
      POSTGRES_DB: ${PG_OLAP_DB}
      DEBEZIUM_OLAP_PASSWORD: ${DEBEZIUM_OLAP_PASSWORD:-debezium_olap_pass}
    ports:
      - "${PG_OLAP_PORT}:5432"
    volumes:
      - pg_olap_data:/var/lib/postgresql/data
      - ./services/postgres/init/03_dwh_schema.sql:/docker-entrypoint-initdb.d/03_dwh_schema.sql:ro
      - ./services/postgres/init/04_dwh_extensions.sql:/docker-entrypoint-initdb.d/04_dwh_extensions.sql:ro
      - ./services/postgres/init/05_meta_views.sql:/docker-entrypoint-initdb.d/05_meta_views.sql:ro
      - ./services/postgres/cdc/olap_debezium_bootstrap.sh:/docker-entrypoint-initdb.d/99_debezium_bootstrap.sh:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${PG_OLAP_USER} -d ${PG_OLAP_DB}"]
      interval: 10s
      timeout: 5s
      retries: 10
    networks:
      - dataops_net

  postgres_metadb:
    image: postgres:${POSTGRES_IMAGE_TAG}
    container_name: postgres_metadb
    restart: unless-stopped
    command:
      - postgres
      - -c
      - wal_level=logical
      - -c
      - max_replication_slots=16
      - -c
      - max_wal_senders=10
    environment:
      POSTGRES_USER: ${PG_META_USER}
      POSTGRES_PASSWORD: ${PG_META_PASSWORD}
      POSTGRES_DB: ${PG_META_DB}
      DEBEZIUM_META_PASSWORD: ${DEBEZIUM_META_PASSWORD:-debezium_meta_pass}
      PG_META_DB: ${PG_META_DB}
    ports:
      - "${PG_META_PORT}:5432"
    volumes:
      - pg_meta_data:/var/lib/postgresql/data
      - ./services/postgres/init/01_create_databases.sql:/docker-entrypoint-initdb.d/01_create_databases.sql:ro
      - ./services/postgres/init/06_generator_meta.sql:/docker-entrypoint-initdb.d/06_generator_meta.sql:ro
      - ./services/postgres/init/07_dbt_rest.sql:/docker-entrypoint-initdb.d/07_dbt_rest.sql:ro
      - ./services/postgres/cdc/metadb_debezium_bootstrap.sh:/docker-entrypoint-initdb.d/99_debezium_bootstrap.sh:ro
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${PG_META_USER} -d ${PG_META_DB}"]
      interval: 10s
      timeout: 5s
      retries: 10
    networks:
      - dataops_net

  redis:
    image: redis:${REDIS_IMAGE_TAG}
    container_name: redis
    restart: unless-stopped
    command: ["redis-server", "/usr/local/etc/redis/redis.conf"]
    ports:
      - "${REDIS_PORT}:6379"
    volumes:
      - redis_data:/data
      - ./services/redis/redis.conf:/usr/local/etc/redis/redis.conf:ro
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 10s
      timeout: 5s
      retries: 10
    networks:
      - dataops_net

  redis_exporter:
    image: oliver006/redis_exporter:${REDIS_EXPORTER_IMAGE_TAG:-v1.64.0}
    container_name: redis_exporter
    restart: unless-stopped
    command:
      - --redis.addr=redis://redis:6379
    depends_on:
      redis:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9121/metrics || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  kafka_exporter:
    image: danielqsj/kafka-exporter:${KAFKA_EXPORTER_IMAGE_TAG:-v1.7.0}
    container_name: kafka_exporter
    restart: unless-stopped
    command:
      - --kafka.server=kafka:9092
    depends_on:
      kafka:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9308/metrics || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  kafka:
    image: confluentinc/cp-kafka:${KAFKA_IMAGE_TAG}
    container_name: kafka
    restart: unless-stopped
    environment:
      KAFKA_NODE_ID: 1
      KAFKA_PROCESS_ROLES: broker,controller
      KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT
      KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
      KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      CLUSTER_ID: MkU3OEVBNTcwNTJENDM2Qk
    ports:
      - "${KAFKA_PORT}:9092"
    volumes:
      - kafka_data:/var/lib/kafka/data
    healthcheck:
      test: ["CMD-SHELL", "kafka-topics --bootstrap-server localhost:9092 --list >/dev/null 2>&1"]
      interval: 20s
      timeout: 10s
      retries: 10
    networks:
      - dataops_net

  minio:
    image: minio/minio:${MINIO_IMAGE_TAG}
    container_name: minio
    restart: unless-stopped
    command: server /data --console-address ":9001"
    environment:
      MINIO_ROOT_USER: ${MINIO_ROOT_USER}
      MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
      MINIO_SERVER_URL: ${MINIO_SERVER_URL:-http://localhost:${MINIO_PORT:-9000}}
      MINIO_BROWSER_REDIRECT_URL: ${MINIO_BROWSER_REDIRECT_URL:-${INGRESS_BASE_URL:-http://localhost:8090}/minio-console/}
    ports:
      - "${MINIO_PORT}:9000"
    volumes:
      - minio_data:/data
      - ./services/minio/config:/root/.minio
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://localhost:9000/minio/health/live >/dev/null"]
      interval: 15s
      timeout: 5s
      retries: 10
    networks:
      - dataops_net

  spark_master:
    image: apache/spark:${SPARK_IMAGE_TAG}
    container_name: spark_master
    hostname: spark-master
    restart: unless-stopped
    command:
      - /opt/spark/bin/spark-class
      - org.apache.spark.deploy.master.Master
      - --host
      - spark-master
      - --port
      - "7077"
      - --webui-port
      - "8080"
    ports:
      - "${SPARK_MASTER_PORT}:7077"
    volumes:
      - spark_master_data:/opt/spark/work-dir
      - ./spark/jobs:/opt/spark/jobs
      - ./spark/conf/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf:ro
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://localhost:8080 >/dev/null"]
      interval: 20s
      timeout: 10s
      retries: 10
    networks:
      - dataops_net

  spark_worker:
    image: apache/spark:${SPARK_IMAGE_TAG}
    container_name: spark_worker
    restart: unless-stopped
    command:
      - /opt/spark/bin/spark-class
      - org.apache.spark.deploy.worker.Worker
      - spark://spark-master:7077
      - --webui-port
      - "8081"
      - --cores
      - "2"
      - --memory
      - 2G
    volumes:
      - spark_worker_data:/opt/spark/work-dir
      - ./spark/jobs:/opt/spark/jobs
      - ./spark/conf/spark-defaults.conf:/opt/spark/conf/spark-defaults.conf:ro
    depends_on:
      spark_master:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://localhost:8081 >/dev/null"]
      interval: 20s
      timeout: 10s
      retries: 10
    networks:
      - dataops_net

  airflow_init:
    build:
      context: .
      dockerfile: services/airflow/Dockerfile
      args:
        AIRFLOW_IMAGE_TAG: ${AIRFLOW_IMAGE_TAG}
    image: dataops/airflow:local
    container_name: airflow_init
    restart: "no"
    environment:
      <<: &airflow_env
        AIRFLOW_HOME: /opt/airflow
        AIRFLOW_CONFIG: /opt/airflow/config/airflow.cfg
        AIRFLOW__CORE__EXECUTOR: LocalExecutor
        AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://${PG_META_USER}:${PG_META_PASSWORD}@postgres_metadb:5432/${AIRFLOW_DB_NAME}
        AIRFLOW__WEBSERVER__SECRET_KEY: ${AIRFLOW_WEBSERVER_SECRET_KEY}
        AIRFLOW__WEBSERVER__BASE_URL: ${INGRESS_BASE_URL:-http://localhost:8090}/airflow
        AIRFLOW__WEBSERVER__ENABLE_PROXY_FIX: "True"
        AIRFLOW__CORE__LOAD_EXAMPLES: "False"
        AIRFLOW__CORE__DAGS_FOLDER: /opt/airflow/pipelines/dags
        PYTHONPATH: /opt/airflow:/opt/airflow/services:/opt/airflow/pipelines:/opt/airflow/spark/common
        DATAOPS_CONFIG_DIR: /opt/airflow/configs/pipeline
        KAFKA_BOOTSTRAP_SERVERS: kafka:9092
        KAFKA_TOPIC_ORDERS: ${KAFKA_TOPIC_ORDERS}
        KAFKA_TOPIC_PAYMENTS: ${KAFKA_TOPIC_PAYMENTS}
        KAFKA_TOPIC_MARKETING_EMAIL: ${KAFKA_TOPIC_MARKETING_EMAIL:-techmart.marketing.email_events}
        KAFKA_TOPIC_SEO_ORGANIC: ${KAFKA_TOPIC_SEO_ORGANIC:-techmart.seo.organic_sessions}
        KAFKA_TOPIC_HR_TIME_TRACKING: ${KAFKA_TOPIC_HR_TIME_TRACKING:-techmart.hr.time_tracking}
        KAFKA_TOPIC_FEATURE_FLAG_EVAL: ${KAFKA_TOPIC_FEATURE_FLAG_EVAL:-techmart.features.evaluated}
        MINIO_ENDPOINT: minio:9000
        MINIO_ROOT_USER: ${MINIO_ROOT_USER}
        MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD}
        MINIO_BUCKET_RAW: ${MINIO_BUCKET_RAW}
        MINIO_PREFIX_PAYMENTS: ${MINIO_PREFIX_PAYMENTS}
        MINIO_PREFIX_RETURNS: ${MINIO_PREFIX_RETURNS}
        MINIO_PREFIX_CATALOG: ${MINIO_PREFIX_CATALOG}
        MINIO_PREFIX_MARKETING_PERF: ${MINIO_PREFIX_MARKETING_PERF:-}
        MINIO_PREFIX_SEO_RANKINGS: ${MINIO_PREFIX_SEO_RANKINGS:-}
        MINIO_PREFIX_TELEMETRY_PERF: ${MINIO_PREFIX_TELEMETRY_PERF:-}
        MINIO_PREFIX_TELEMETRY_ERRORS: ${MINIO_PREFIX_TELEMETRY_ERRORS:-}
        MINIO_PREFIX_HR_PERF: ${MINIO_PREFIX_HR_PERF:-}
        DBT_REST_BASE_URL: ${DBT_REST_BASE_URL:-http://dbt-rest:8580}
        DBT_REST_TOKEN: ${DBT_REST_TOKEN:-}
        PROMETHEUS_PUSHGATEWAY_URL: ${PROMETHEUS_PUSHGATEWAY_URL:-http://pushgateway:9091}
        DWH_JDBC_URL: jdbc:postgresql://postgres_olap:5432/${PG_OLAP_DB}
        DWH_JDBC_USER: ${PG_OLAP_USER}
        DWH_JDBC_PASSWORD: ${PG_OLAP_PASSWORD}
        SPARK_PRIVACY_SALT: ${SPARK_PRIVACY_SALT:-dev-not-for-production}
        SPARK_JOB_ENV: ${SPARK_JOB_ENV:-production}
        MLFLOW_TRACKING_URI: ${MLFLOW_TRACKING_URI}
        MLFLOW_EXPERIMENT_NAME: ${MLFLOW_EXPERIMENT_NAME}
        MLFLOW_MODEL_NAME: ${MLFLOW_MODEL_NAME}
        MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL}
        AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
        AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
        ATLAS_REST_URL: ${ATLAS_REST_URL:-http://atlas_server:21000/}
        DEBEZIUM_CONNECT_HEALTH_URL: ${DEBEZIUM_CONNECT_HEALTH_URL:-http://debezium_connect:8083/}
        PG_REFLECT_DSN: postgresql://${PG_OLTP_USER}:${PG_OLTP_PASSWORD}@postgres_oltp:5432/${PG_OLTP_DB}
        ATLAS_QUALITY_MIN_TOTAL: ${ATLAS_QUALITY_MIN_TOTAL:-5}
        GENERATOR_CONFIG_JSON: ${GENERATOR_CONFIG_JSON:-/opt/airflow/configs/generators/company.generator.json}
        ATLAS_PUBLISH_RETRIES: ${ATLAS_PUBLISH_RETRIES:-3}
        ATLAS_PUBLISH_RETRY_DELAY_SEC: ${ATLAS_PUBLISH_RETRY_DELAY_SEC:-5}
        ATLAS_QUALITY_RETRIES: ${ATLAS_QUALITY_RETRIES:-2}
        ATLAS_QUALITY_RETRY_DELAY_SEC: ${ATLAS_QUALITY_RETRY_DELAY_SEC:-15}
        ATLAS_SKIP_STATIC_KAFKA: ${ATLAS_SKIP_STATIC_KAFKA:-}
        PIP_DEFAULT_TIMEOUT: ${PIP_DEFAULT_TIMEOUT:-180}
        PIP_RETRIES: ${PIP_RETRIES:-10}
        PIP_DISABLE_PIP_VERSION_CHECK: "1"
      AIRFLOW_ADMIN_USER: ${AIRFLOW_ADMIN_USER}
      AIRFLOW_ADMIN_PASSWORD: ${AIRFLOW_ADMIN_PASSWORD}
    user: "${AIRFLOW_UID}:0"
    command: >
      bash -c "
        airflow db migrate &&
        airflow users create --role Admin --username $$AIRFLOW_ADMIN_USER --password $$AIRFLOW_ADMIN_PASSWORD --firstname Data --lastname Ops --email admin@example.com &&
        airflow connections import /opt/airflow/configs/airflow/connections.json --overwrite
      "
    volumes: &airflow_volumes
      - ./pipelines:/opt/airflow/pipelines
      - ./pipelines/dags:/opt/airflow/dags
      - ./pipelines/plugins:/opt/airflow/plugins
      - ./services:/opt/airflow/services
      - ./spark:/opt/airflow/spark
      - ./ml:/opt/airflow/ml
      - ./generators:/opt/airflow/generators
      - ./configs:/opt/airflow/configs
      - ./services/airflow/config:/opt/airflow/config
      - ./dbt:/workspace/dbt
      - ./infra/metadata/atlas:/opt/airflow/infra/metadata/atlas
      - airflow_logs:/opt/airflow/logs
    depends_on:
      postgres_metadb:
        condition: service_healthy
    networks:
      - dataops_net

  airflow_scheduler:
    image: dataops/airflow:local
    container_name: airflow_scheduler
    restart: unless-stopped
    environment:
      <<: *airflow_env
    user: "${AIRFLOW_UID}:0"
    command: ["airflow", "scheduler"]
    volumes: *airflow_volumes
    depends_on:
      airflow_init:
        condition: service_completed_successfully
      dbt_rest:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "airflow jobs check --job-type SchedulerJob --local"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  airflow_triggerer:
    image: dataops/airflow:local
    container_name: airflow_triggerer
    restart: unless-stopped
    environment:
      <<: *airflow_env
    user: "${AIRFLOW_UID}:0"
    command: ["airflow", "triggerer"]
    volumes: *airflow_volumes
    depends_on:
      airflow_init:
        condition: service_completed_successfully
      dbt_rest:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "airflow jobs check --job-type TriggererJob --local"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  airflow_webserver:
    image: dataops/airflow:local
    container_name: airflow_webserver
    restart: unless-stopped
    environment:
      <<: *airflow_env
    user: "${AIRFLOW_UID}:0"
    command: ["airflow", "webserver"]
    volumes: *airflow_volumes
    depends_on:
      airflow_init:
        condition: service_completed_successfully
      dbt_rest:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8080/airflow/health')\""]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  dbt:
    image: python:3.11-slim
    container_name: dbt
    restart: unless-stopped
    command: ["/bin/bash", "-c", "pip install --no-cache-dir dbt-postgres==${DBT_IMAGE_TAG} && sleep infinity"]
    environment:
      DBT_PROFILES_DIR: /workspace/dbt
    volumes:
      - ./dbt:/workspace/dbt
    depends_on:
      postgres_olap:
        condition: service_healthy
    networks:
      - dataops_net

  dbt_rest:
    build:
      context: ./services/dbt_rest
      dockerfile: Dockerfile
      args:
        DBT_VERSION: ${DBT_IMAGE_TAG:-1.8.2}
    image: dataops/dbt-rest:local
    container_name: dbt-rest
    restart: unless-stopped
    environment:
      DBT_PROJECT_DIR: /workspace/dbt
      DBT_PROFILES_DIR: /workspace/dbt
      DBT_REST_TOKEN: ${DBT_REST_TOKEN:-}
      DBT_REST_DB_DSN: postgresql://${PG_META_USER}:${PG_META_PASSWORD}@postgres_metadb:5432/${PG_META_DB}
    volumes:
      - ./dbt:/workspace/dbt
    depends_on:
      postgres_olap:
        condition: service_healthy
      postgres_metadb:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8580/health')\""]
      interval: 15s
      timeout: 10s
      retries: 8
      start_period: 90s
    networks:
      dataops_net:
        aliases:
          - dbt-rest

  mlflow:
    image: ghcr.io/mlflow/mlflow:v2.15.1
    container_name: mlflow
    restart: unless-stopped
    command:
      - /bin/sh
      - -c
      - >
        pip install --no-cache-dir psycopg2-binary==2.9.9 boto3==1.34.155 &&
        mlflow server
        --host 0.0.0.0
        --port 5000
        --static-prefix /mlflow
        --backend-store-uri ${MLFLOW_BACKEND_STORE_URI}
        --artifacts-destination ${MLFLOW_ARTIFACT_ROOT}
        --serve-artifacts
    environment:
      MLFLOW_S3_ENDPOINT_URL: ${MLFLOW_S3_ENDPOINT_URL}
      AWS_ACCESS_KEY_ID: ${MINIO_ROOT_USER}
      AWS_SECRET_ACCESS_KEY: ${MINIO_ROOT_PASSWORD}
      AWS_DEFAULT_REGION: ${AWS_DEFAULT_REGION}
    depends_on:
      postgres_metadb:
        condition: service_healthy
      minio:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:5000/health')\""]
      interval: 20s
      timeout: 10s
      retries: 10
    networks:
      - dataops_net

  superset_init:
    build:
      context: ./services/superset
      dockerfile: Dockerfile
      args:
        SUPERSET_IMAGE_TAG: ${SUPERSET_IMAGE_TAG:-latest}
    image: techmart/superset:dataops-local
    container_name: superset_init
    restart: "no"
    environment:
      INGRESS_BASE_URL: ${INGRESS_BASE_URL:-http://localhost:8090}
      SUPERSET_APP_ROOT: /superset
      SUPERSET_WEBSERVER_BASE_URL: ${INGRESS_BASE_URL:-http://localhost:8090}/superset/
      SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY}
      SUPERSET_CONFIG_PATH: /app/pythonpath/superset_config.py
      SUPERSET_ADMIN_USERNAME: ${SUPERSET_ADMIN_USER}
      SUPERSET_ADMIN_PASSWORD: ${SUPERSET_ADMIN_PASSWORD}
      SUPERSET_ADMIN_FIRSTNAME: Data
      SUPERSET_ADMIN_LASTNAME: Ops
      SUPERSET_ADMIN_EMAIL: admin@example.com
      SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://${PG_META_USER}:${PG_META_PASSWORD}@postgres_metadb:5432/${SUPERSET_DB_NAME:-superset}
      SUPERSET_DWH_DATABASE_URI: postgresql+psycopg2://${PG_OLAP_USER}:${PG_OLAP_PASSWORD}@postgres_olap:5432/${PG_OLAP_DB:-techmart_dwh}
      PYTHONPATH: /home/superset/.local/lib/python3.10/site-packages
    command: >
      /bin/sh -c "
      superset db upgrade &&
      superset fab create-admin --username $$SUPERSET_ADMIN_USERNAME --password $$SUPERSET_ADMIN_PASSWORD --firstname $$SUPERSET_ADMIN_FIRSTNAME --lastname $$SUPERSET_ADMIN_LASTNAME --email $$SUPERSET_ADMIN_EMAIL &&
      superset init &&
      python /app/pythonpath/superset_bootstrap.py
      "
    volumes:
      - ./configs/app/superset_config.py:/app/pythonpath/superset_config.py:ro
      - ./configs/app/superset_bootstrap.py:/app/pythonpath/superset_bootstrap.py:ro
      - superset_home:/app/superset_home
    depends_on:
      postgres_metadb:
        condition: service_healthy
      postgres_olap:
        condition: service_healthy
    networks:
      - dataops_net

  superset:
    image: techmart/superset:dataops-local
    container_name: superset
    restart: unless-stopped
    environment:
      INGRESS_BASE_URL: ${INGRESS_BASE_URL:-http://localhost:8090}
      SUPERSET_APP_ROOT: /superset
      SUPERSET_WEBSERVER_BASE_URL: ${INGRESS_BASE_URL:-http://localhost:8090}/superset/
      SUPERSET_SECRET_KEY: ${SUPERSET_SECRET_KEY}
      SUPERSET_CONFIG_PATH: /app/pythonpath/superset_config.py
      SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://${PG_META_USER}:${PG_META_PASSWORD}@postgres_metadb:5432/${SUPERSET_DB_NAME:-superset}
      SUPERSET_DWH_DATABASE_URI: postgresql+psycopg2://${PG_OLAP_USER}:${PG_OLAP_PASSWORD}@postgres_olap:5432/${PG_OLAP_DB:-techmart_dwh}
      PYTHONPATH: /home/superset/.local/lib/python3.10/site-packages
    command: ["/bin/sh", "-c", "superset run -h 0.0.0.0 -p 8088"]
    volumes:
      - ./configs/app/superset_config.py:/app/pythonpath/superset_config.py:ro
      - ./configs/app/superset_bootstrap.py:/app/pythonpath/superset_bootstrap.py:ro
      - superset_home:/app/superset_home
    depends_on:
      postgres_metadb:
        condition: service_healthy
      superset_init:
        condition: service_completed_successfully
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8088/superset/health')\"",
        ]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  jupyterhub:
    image: jupyter/datascience-notebook:${JUPYTER_IMAGE_TAG}
    container_name: jupyterhub
    restart: unless-stopped
    environment:
      JUPYTER_TOKEN: ${JUPYTER_TOKEN}
      JUPYTER_ENABLE_LAB: "yes"
    command:
      [
        "start-notebook.py",
        "--NotebookApp.token=${JUPYTER_TOKEN}",
        "--NotebookApp.base_url=/jupyter/",
        "--ServerApp.base_url=/jupyter/",
        "--NotebookApp.allow_origin=*",
      ]
    volumes:
      - jupyterhub_data:/home/jovyan
      - ./notebooks:/home/jovyan/work
      - ./:/workspace
    healthcheck:
      test: ["CMD-SHELL", "python -c \"import urllib.request; urllib.request.urlopen('http://localhost:8888/jupyter/')\""]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  pushgateway:
    image: prom/pushgateway:${PUSHGATEWAY_IMAGE_TAG:-v1.9.0}
    container_name: pushgateway
    restart: unless-stopped
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9091/-/healthy || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  cadvisor:
    image: gcr.io/cadvisor/cadvisor:v0.49.1
    container_name: cadvisor
    restart: unless-stopped
    privileged: true
    volumes:
      - /:/rootfs:ro
      - /var/run:/var/run:ro
      - /sys:/sys:ro
      - /var/lib/docker/:/var/lib/docker:ro
      - /dev/disk/:/dev/disk:ro
    networks:
      - dataops_net
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:8080/healthz || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5

  prometheus:
    image: prom/prometheus:${PROMETHEUS_IMAGE_TAG}
    container_name: prometheus
    restart: unless-stopped
    command:
      - --config.file=/etc/prometheus/prometheus.yml
      - --storage.tsdb.path=/prometheus
      - --web.enable-lifecycle
    volumes:
      - ./infra/monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro
      - ./infra/monitoring/prometheus-rules:/etc/prometheus/rules:ro
      - prometheus_data:/prometheus
    depends_on:
      pushgateway:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:9090/-/healthy || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  data_generator:
    build:
      context: .
      dockerfile: generators/Dockerfile
    image: techmart/data-generator:local
    container_name: data_generator
    restart: unless-stopped
    profiles: ["generators"]
    volumes:
      - ./configs/generators:/app/configs/generators:ro
      - ./services/postgres/init/02b_oltp_marketing_hr_finance.sql:/app/sql/02b_oltp_marketing_hr_finance.sql:ro
      - ./services/postgres/init/02c_oltp_retail_legacy.sql:/app/sql/02c_oltp_retail_legacy.sql:ro
    environment:
      GENERATOR_CONFIG_DIR: ${GENERATOR_CONFIG_DIR:-/app/configs/generators}
      GENERATOR_SETTINGS_FILE: ${GENERATOR_SETTINGS_FILE:-/app/configs/generators/company.generator.json}
      GENERATOR_SETTINGS_JSON: ${GENERATOR_SETTINGS_JSON:-}
      GENERATOR_MODE: ${GENERATOR_MODE:-}
      GENERATOR_TICK_SECONDS: ${GENERATOR_TICK_SECONDS:-}
      GENERATOR_SEED: ${GENERATOR_SEED:-}
      GENERATOR_SEED_USERS: ${GENERATOR_SEED_USERS:-}
      GENERATOR_SEED_SELLERS: ${GENERATOR_SEED_SELLERS:-}
      GENERATOR_SEED_PRODUCTS: ${GENERATOR_SEED_PRODUCTS:-}
      GENERATOR_ORDERS_PER_TICK_MIN: ${GENERATOR_ORDERS_PER_TICK_MIN:-}
      GENERATOR_ORDERS_PER_TICK_MAX: ${GENERATOR_ORDERS_PER_TICK_MAX:-}
      GENERATOR_CLICKS_PER_TICK_MIN: ${GENERATOR_CLICKS_PER_TICK_MIN:-}
      GENERATOR_CLICKS_PER_TICK_MAX: ${GENERATOR_CLICKS_PER_TICK_MAX:-}
      GENERATOR_MINIO_BATCH_TICKS: ${GENERATOR_MINIO_BATCH_TICKS:-}
      GENERATOR_ENABLE_OLTP: ${GENERATOR_ENABLE_OLTP:-}
      GENERATOR_ENABLE_KAFKA: ${GENERATOR_ENABLE_KAFKA:-}
      GENERATOR_ENABLE_REDIS: ${GENERATOR_ENABLE_REDIS:-}
      GENERATOR_ENABLE_MINIO: ${GENERATOR_ENABLE_MINIO:-}
      GENERATOR_ENABLE_EXTENSIONS: ${GENERATOR_ENABLE_EXTENSIONS:-}
      GENERATOR_USE_META_STORE: ${GENERATOR_USE_META_STORE:-false}
      GENERATOR_META_DSN: ${GENERATOR_META_DSN:-}
      GENERATOR_PROFILE: ${GENERATOR_PROFILE:-default}
      OLTP_DSN: postgresql://${PG_OLTP_USER}:${PG_OLTP_PASSWORD}@postgres_oltp:5432/${PG_OLTP_DB}
      OLTP_EXTENSIONS_SQL: ${OLTP_EXTENSIONS_SQL:-/app/sql/02b_oltp_marketing_hr_finance.sql}
      KAFKA_BOOTSTRAP_SERVERS: ${KAFKA_BOOTSTRAP_SERVERS:-kafka:9092}
      KAFKA_TOPIC_CLICKSTREAM: ${KAFKA_TOPIC_CLICKSTREAM:-}
      KAFKA_TOPIC_ORDERS: ${KAFKA_TOPIC_ORDERS:-}
      KAFKA_TOPIC_PAYMENTS: ${KAFKA_TOPIC_PAYMENTS:-}
      KAFKA_TOPIC_SHIPMENTS: ${KAFKA_TOPIC_SHIPMENTS:-}
      KAFKA_TOPIC_MARKETING_EMAIL: ${KAFKA_TOPIC_MARKETING_EMAIL:-techmart.marketing.email_events}
      KAFKA_TOPIC_SEO_ORGANIC: ${KAFKA_TOPIC_SEO_ORGANIC:-techmart.seo.organic_sessions}
      KAFKA_TOPIC_HR_TIME_TRACKING: ${KAFKA_TOPIC_HR_TIME_TRACKING:-techmart.hr.time_tracking}
      KAFKA_TOPIC_FEATURE_FLAG_EVAL: ${KAFKA_TOPIC_FEATURE_FLAG_EVAL:-techmart.features.evaluated}
      REDIS_URL: ${REDIS_URL:-redis://redis:6379/0}
      REDIS_CHANNEL_ORDERS: ${REDIS_CHANNEL_ORDERS:-}
      REDIS_CHANNEL_PAYMENTS: ${REDIS_CHANNEL_PAYMENTS:-}
      REDIS_CHANNEL_CLICKSTREAM: ${REDIS_CHANNEL_CLICKSTREAM:-}
      REDIS_STREAM_ORDERS: ${REDIS_STREAM_ORDERS:-}
      REDIS_KEY_WEB_VITALS_PREFIX: ${REDIS_KEY_WEB_VITALS_PREFIX:-}
      MINIO_ENDPOINT: ${MINIO_ENDPOINT:-minio:9000}
      MINIO_ACCESS_KEY: ${MINIO_ROOT_USER}
      MINIO_SECRET_KEY: ${MINIO_ROOT_PASSWORD}
      MINIO_BUCKET_RAW: ${MINIO_BUCKET_RAW:-}
      MINIO_PREFIX_PAYMENTS: ${MINIO_PREFIX_PAYMENTS:-}
      MINIO_PREFIX_RETURNS: ${MINIO_PREFIX_RETURNS:-}
      MINIO_PREFIX_CATALOG: ${MINIO_PREFIX_CATALOG:-}
      MINIO_PREFIX_MARKETING_PERF: ${MINIO_PREFIX_MARKETING_PERF:-}
      MINIO_PREFIX_SEO_RANKINGS: ${MINIO_PREFIX_SEO_RANKINGS:-}
      MINIO_PREFIX_TELEMETRY_PERF: ${MINIO_PREFIX_TELEMETRY_PERF:-}
      MINIO_PREFIX_TELEMETRY_ERRORS: ${MINIO_PREFIX_TELEMETRY_ERRORS:-}
      MINIO_PREFIX_HR_PERF: ${MINIO_PREFIX_HR_PERF:-}
    working_dir: /app
    command: ["python", "-u", "generator.py"]
    depends_on:
      postgres_oltp:
        condition: service_healthy
      kafka:
        condition: service_healthy
      redis:
        condition: service_healthy
      minio:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "test -f /tmp/generator.alive"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  grafana:
    image: grafana/grafana:${GRAFANA_IMAGE_TAG}
    container_name: grafana
    restart: unless-stopped
    environment:
      GF_SECURITY_ADMIN_USER: ${GRAFANA_ADMIN_USER}
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_ADMIN_PASSWORD}
      GF_PATHS_PROVISIONING: /etc/grafana/provisioning
      GF_SERVER_ROOT_URL: ${INGRESS_BASE_URL:-http://localhost:8090}/grafana/
      GF_SERVER_SERVE_FROM_SUB_PATH: "true"
    volumes:
      - grafana_data:/var/lib/grafana
      - ./infra/monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
      - ./infra/monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
    depends_on:
      postgres_olap:
        condition: service_healthy
      prometheus:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://localhost:3000/api/health || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

  schema_registry:
    image: confluentinc/cp-schema-registry:${CP_SCHEMA_REGISTRY_TAG:-7.6.1}
    hostname: schema_registry
    container_name: schema_registry
    restart: unless-stopped
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema_registry
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: kafka:9092
      SCHEMA_REGISTRY_DEBUG: "true"
    depends_on:
      kafka:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://localhost:8081/subjects >/dev/null"]
      interval: 20s
      timeout: 10s
      retries: 20
      start_period: 45s
    networks:
      - dataops_net

  debezium_connect:
    image: debezium/connect:${DEBEZIUM_CONNECT_TAG:-2.5}
    hostname: debezium_connect
    container_name: debezium_connect
    restart: unless-stopped
    depends_on:
      kafka:
        condition: service_healthy
      schema_registry:
        condition: service_healthy
    environment:
      BOOTSTRAP_SERVERS: kafka:9092
      GROUP_ID: ${DEBEZIUM_GROUP_ID:-debezium-cluster}
      CONFIG_STORAGE_TOPIC: debezium_connect_configs
      OFFSET_STORAGE_TOPIC: debezium_connect_offsets
      STATUS_STORAGE_TOPIC: debezium_connect_status
      CONFIG_STORAGE_REPLICATION_FACTOR: 1
      OFFSET_STORAGE_REPLICATION_FACTOR: 1
      STATUS_STORAGE_REPLICATION_FACTOR: 1
      KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_PLUGIN_PATH: /kafka/connect
    healthcheck:
      test: ["CMD-SHELL", "curl -fsS http://localhost:8083/connectors >/dev/null 2>&1"]
      interval: 20s
      timeout: 10s
      retries: 30
      start_period: 90s
    networks:
      - dataops_net

  atlas_server:
    image: sburn/apache-atlas:${SBURN_APACHE_ATLAS_TAG:-latest}
    container_name: atlas_server
    restart: unless-stopped
    environment:
      TZ: UTC
      ATLAS_JAVA_OPTS: ${ATLAS_JAVA_OPTS:- -Xmx3G -Xms1G}
    healthcheck:
      test: ["CMD-SHELL", "bash -lc 'exec 3<>/dev/tcp/127.0.0.1/21000'"]
      interval: 45s
      timeout: 15s
      retries: 30
      start_period: 420s
    networks:
      - dataops_net

  node_red:
    build:
      context: ./services/node_red
      dockerfile: Dockerfile
      args:
        NODE_RED_IMAGE_TAG: ${NODE_RED_IMAGE_TAG:-4.1.8-22}
    image: dataops/node-red:local
    container_name: node_red
    restart: unless-stopped
    environment:
      NODE_RED_ADMIN_USER: ${NODE_RED_ADMIN_USER:-admin}
      NODE_RED_ADMIN_PASSWORD: ${NODE_RED_ADMIN_PASSWORD:-changeme}
    volumes:
      - node_red_data:/data
    healthcheck:
      test:
        [
          "CMD",
          "node",
          "-e",
          "require('http').get('http://127.0.0.1:1880/node-red/',r=>process.exit([200,401,302,304].includes(r.statusCode)?0:1)).on('error',()=>process.exit(1))",
        ]
      interval: 20s
      timeout: 10s
      retries: 8
      start_period: 40s
    networks:
      - dataops_net

  # NL2SQL HR: HTTP UI/API on container port 8060 (ingress /nl2sql/ -> nl2sql_app:8060; Prometheus scrapes :8060/metrics).
  nl2sql_app:
    build:
      context: .
      dockerfile: services/nl2sql_app/Dockerfile
    image: dataops/nl2sql-app:local
    container_name: nl2sql_app
    restart: unless-stopped
    environment:
      DB_URL: ${NL2SQL_DB_URL:-postgresql+psycopg2://${PG_OLAP_USER}:${PG_OLAP_PASSWORD}@postgres_olap:5432/${PG_OLAP_DB}}
      MLFLOW_TRACKING_URI: ${MLFLOW_TRACKING_URI:-http://mlflow:5000}
      NL2SQL_MODEL_URI: ${NL2SQL_MODEL_URI:-models:/nl2sql_qwen/latest}
      NL2SQL_MODEL_NAME: ${NL2SQL_MODEL_NAME:-nl2sql_qwen}
      NL2SQL_AUTO_LOG_MODEL: ${NL2SQL_AUTO_LOG_MODEL:-false}
      NL2SQL_BASE_MODEL_ID: ${NL2SQL_BASE_MODEL_ID:-Qwen/Qwen2.5-3B-Instruct}
      NL2SQL_EMBEDDING_MODEL: ${NL2SQL_EMBEDDING_MODEL:-sentence-transformers/all-MiniLM-L6-v2}
      NL2SQL_RAG_TOP_K: ${NL2SQL_RAG_TOP_K:-3}
      NL2SQL_MAX_NEW_TOKENS: ${NL2SQL_MAX_NEW_TOKENS:-256}
      NL2SQL_TEMPERATURE: ${NL2SQL_TEMPERATURE:-0.0}
      NL2SQL_MODEL_DEVICE: ${NL2SQL_MODEL_DEVICE:-cpu}
      NL2SQL_DEBUG: ${NL2SQL_DEBUG:-false}
      NL2SQL_EXPOSE_METRICS: ${NL2SQL_EXPOSE_METRICS:-true}
    depends_on:
      postgres_olap:
        condition: service_healthy
      mlflow:
        condition: service_healthy
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8060/health', timeout=3)\"",
        ]
      interval: 20s
      timeout: 10s
      retries: 5
      start_period: 60s
    networks:
      - dataops_net

  portal_web:
    build:
      context: .
      dockerfile: services/portal_web/Dockerfile
    container_name: portal_web
    restart: unless-stopped
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock:ro
    networks:
      - dataops_net
    healthcheck:
      test:
        [
          "CMD-SHELL",
          "python -c \"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8050/health', timeout=3)\"",
        ]
      interval: 20s
      timeout: 10s
      retries: 5
      start_period: 20s

  ingress:
    image: nginx:1.27-alpine
    container_name: dataops_ingress
    restart: unless-stopped
    ports:
      - "80:80"
      - "${INGRESS_PORT:-8090}:80"
    volumes:
      - ./infra/ingress/nginx.conf:/etc/nginx/nginx.conf:ro
      - ./dbt/target:/var/www/dbt:ro
    depends_on:
      portal_web:
        condition: service_healthy
      airflow_webserver:
        condition: service_healthy
      mlflow:
        condition: service_healthy
      grafana:
        condition: service_healthy
      superset:
        condition: service_healthy
      jupyterhub:
        condition: service_healthy
      prometheus:
        condition: service_healthy
      pushgateway:
        condition: service_healthy
      spark_master:
        condition: service_healthy
      spark_worker:
        condition: service_healthy
      minio:
        condition: service_healthy
      schema_registry:
        condition: service_healthy
      debezium_connect:
        condition: service_healthy
      atlas_server:
        condition: service_started
      node_red:
        condition: service_healthy
      nl2sql_app:
        condition: service_healthy
    healthcheck:
      test: ["CMD-SHELL", "wget --no-verbose --tries=1 --spider http://127.0.0.1/ || exit 1"]
      interval: 30s
      timeout: 10s
      retries: 5
    networks:
      - dataops_net

networks:
  dataops_net:
    driver: bridge

volumes:
  pg_oltp_data:
  pg_olap_data:
  pg_meta_data:
  redis_data:
  kafka_data:
  minio_data:
  spark_master_data:
  spark_worker_data:
  airflow_logs:
  superset_home:
  jupyterhub_data:
  prometheus_data:
  grafana_data:
  node_red_data: