docker-compose.yaml

version: '3'

services:
  scylla:
    image: scylladb/scylla:latest
    networks:
      - scylla
    volumes:
      - ./data/scylla:/var/lib/scylla
    ports:
      - "8000:8000"
    command: "--smp 2 --memory 2048M --alternator-port 8000 --alternator-write-isolation always_use_lwt"

  cassandra:
    image: cassandra:latest
    networks:
      - scylla
    volumes:
      - ./data/cassandra:/var/lib/cassandra

  spark-master:
    image: bde2020/spark-master:2.4.4-hadoop2.7
    container_name: spark-master
    hostname: spark-master
    environment:
      INIT_DAEMON_STEP: setup_spark
      SPARK_MASTER: spark://spark-master:7077
      SPARK_CONF_DIR: /conf
      SPARK_PUBLIC_DNS: spark-master
    networks:
      - scylla
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7006
      - 7077
      - 6066
    ports:
      - 4040:4040
      - 6066:6066
      - 7077:7077
      - 8080:8080
    volumes:
      - ./data/spark-master:/tmp/data
      - ./target/scala-2.11:/jars
      - ./parquet-data:/parquet-data
      - ./:/app

  spark-worker:
    image: bde2020/spark-worker:2.4.4-hadoop2.7
    hostname: spark-worker
    container_name: spark-worker-1
    environment:
      SPARK_WORKER_CORES: 3
      SPARK_WORKER_MEMORY: 1024m
      SPARK_WORKER_WEBUI_PORT: 8081
      SPARK_PUBLIC_DNS: spark-worker
    networks:
      - scylla
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 7016
      - 8881
    ports:
      - 8081:8081
    volumes:
      - ./data/spark-worker:/tmp/data
      - ./parquet-data:/parquet-data
    depends_on:
      - spark-master

networks:
  scylla: