docker-compose.yml

version: "3.7"
services:

  traefik-proxy:
      image: "traefik:v2.2"
      container_name: traefik_proxy
      command:
        - "--api.insecure=true"
        - "--providers.docker=true"
        - "--providers.docker.exposedbydefault=false"
      labels:
        - "traefik.enable=true"
        - "traefik.http.routers.api.rule=Host(`traefik.localhost`)"
        - "traefik.http.routers.api.service=api@internal"
        - "traefik.http.routers.api.entrypoints=http"
      ports:
        - "80:80"
        - "8080:8080"
      volumes:
        - /var/run/docker.sock:/var/run/docker.sock:ro

  master:
    build: docker/.
    command: bin/spark-class org.apache.spark.deploy.master.Master -h master
    container_name: master_spark
    environment:
      SPARK_CONF_DIR: /tmp/conf
      SPARK_MASTER_WEBUI_PORT: 8082
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.master.rule=Host(`master.localhost`)"
      - "traefik.http.routers.master.entrypoints=http"
      - "traefik.http.services.master.loadbalancer.server.port=8082"
    expose:
      - 7077
      - 4040
    depends_on:
      - traefik-proxy
    ports:
      - "7077:7077"
      - "4040:4040"
    volumes:
      - ./conf/master:/tmp/conf
      - ./src:/tmp/src
      - ./data:/tmp/data
      - ./logs:/tmp/logs

  worker:
    build: docker/.
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://master:7077
    environment:
      SPARK_CONF_DIR: /tmp/conf
      SPARK_WORKER_CORES: 2
      SPARK_WORKER_MEMORY: 1g
    links:
      - master
    expose:
      - 8081
    volumes:
      - ./conf/worker:/tmp/conf
      - ./logs:/tmp/logs
      - ./data:/tmp/data

  history:
    build: docker/.
    command: bin/spark-class org.apache.spark.deploy.history.HistoryServer
    container_name: history_spark
    environment:
      SPARK_CONF_DIR: /tmp/conf
    links:
      - master
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.history.rule=Host(`history.localhost`)"
      - "traefik.http.routers.history.entrypoints=http"
      - "traefik.http.services.history.loadbalancer.server.port=18080"
    expose:
      - 18080
    volumes:
      - ./conf/master:/tmp/conf
      - ./logs:/tmp/logs

  zeppelin:
    build: docker/.
    command: /usr/zeppelin/bin/zeppelin.sh
    container_name: zeppelin_spark
    environment:
      ZEPPELIN_NOTEBOOK_DIR: /usr/zeppelin/notebook
      ZEPPELIN_PORT: 8090
      ZEPPELIN_LOG_DIR: /usr/zeppelin/logs
      ZEPPELIN_CONF_DIR: /usr/zeppelin/conf
      ZEPPELIN_ADDR: 0.0.0.0
    links:
      - master
    labels:
      - "traefik.enable=true"
      - "traefik.http.routers.zeppelin.rule=Host(`zeppelin.localhost`)"
      - "traefik.http.routers.zeppelin.entrypoints=http"
      - "traefik.http.services.zeppelin.loadbalancer.server.port=8090"
    expose:
      - 8090
    volumes:
      - ./notebook:/usr/zeppelin/notebook
      - ./data:/tmp/data