docker-compose.yaml

services:
  search-engine:
    build: ./search
    ports:
      - 5000:5000
    volumes:
      - ./data:/usr/src/data

  web:
    platform: "linux/amd64"
    build:
      context: ./web
    ports:
      - 3010:3010
      - 3011:3011
    environment:
      DB_HOST: "rethinkdb"
      DB_PORT: "28015"
      DB_NAME: "football"
      PORT: "3010"
      SEARCH_ENGINE_URL: "http://search-engine:5000"
    networks:
      - rethinkdb-net

  namenode:
    build:
      context: ./hadoop
    hostname: namenode
    command: ["bash", "-c", "./start.sh"]
    ports:
      - 9870:9870
    environment:
      ENSURE_NAMENODE_DIR: "/tmp/hadoop-root/dfs/name"
    volumes:
      - ./hadoop/config:/opt/hadoop/etc/hadoop
      - ./data/football-events:/opt/hadoop/data
      - ./hadoop/scripts/start-namenode.sh:/opt/hadoop/start.sh
    networks:
      - hadoop-net

  datanode:
    depends_on:
      - namenode
    build:
      context: ./hadoop
    command: ["hdfs", "datanode"]
    volumes:
      - ./hadoop/config:/opt/hadoop/etc/hadoop
    networks:
      - hadoop-net

  resourcemanager:
    build:
      context: ./hadoop
    hostname: resourcemanager
    command: ["yarn", "resourcemanager"]
    ports:
      - 8088:8088
    volumes:
      - ./hadoop/config:/opt/hadoop/etc/hadoop
    networks:
      - hadoop-net

  nodemanager:
    build:
      context: ./hadoop
    command: ["yarn", "nodemanager"]
    volumes:
      - ./hadoop/config:/opt/hadoop/etc/hadoop
    networks:
      - hadoop-net

  rethinkdb:
    image: rethinkdb
    ports:
      - 8080:8080
      - 28015:28015
      - 29015:29015
    networks:
      - rethinkdb-net

  spark-builder:
    build:
      context: ./spark
      dockerfile: Dockerfile.builder
    volumes:
      - ./spark/src:/opt/spark/work-dir/spark-apps
      - ./.build:/opt/spark/work-dir/build

  spark:
    depends_on:
      - spark-builder
      - namenode
      - datanode
    image: apache/spark-py
    environment:
      YARN_CONF_DIR: "/opt/spark/conf/hadoop"
      SPARK_CONF_DIR: "/opt/spark/conf/spark"
      KAFKA_BOOTSTRAP_SERVERS: "kafka-broker:9092"
      KAFKA_TOPIC: "stream-football"
    volumes:
      - ./.build:/opt/spark/work-dir/python-packages
      - ./spark/src:/opt/spark/work-dir/spark-apps/src
      - ./spark/main.py:/opt/spark/work-dir/spark-apps/main.py
      - ./spark/config:/opt/spark/conf/spark
      - ./hadoop/config:/opt/spark/conf/hadoop
    ports:
      - 4040:4040
    command: ["/opt/spark/bin/spark-submit", "spark-apps/main.py"]
    networks:
      - rethinkdb-net
      - hadoop-net
      - kafka-net

  scraper:
    build: ./scraper
    env_file:
      - ./scraper/.env
    depends_on:
      - kafka-broker
    networks:
      - kafka-net

  kafka-broker:
    image: apache/kafka:3.7.0
    ports:
      - 9092:9092
      - 9093:9093
    volumes:
      - ./kafka/config:/mnt/shared/config
    environment:
      CLUSTER_ID: "4L6g3nShT-eMCtK--X86sw"
      KAFKA_NODE_ID: 1
      KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka-broker:29093"
      KAFKA_LISTENERS: "PLAINTEXT_HOST://:9092,SSL://:9093,CONTROLLER://:29093,PLAINTEXT://:19092"
    networks:
      - kafka-net

  datasets-downloader:
    profiles:
      - datasets
    build: ./data
    volumes:
      - ./data:/data # Mount host directory `./data` to container's `/data`
      - $HOME/.kaggle:/root/.kaggle:ro # Mount Kaggle credentials as read-only

networks:
  kafka-net:
    driver: bridge
  rethinkdb-net:
    driver: bridge
  hadoop-net:
    driver: bridge