docker-compose.yml

networks:
  aqm:
    name: aqm
    ipam:
      driver: default
      config:
        - subnet: 10.0.100.0/24

services:
  kafka_zk:
    build:
      context: kafka/
      dockerfile: Dockerfile
    container_name: kafkaZK
    environment:
      KAFKA_ACTION: start-zk
    networks:
      aqm:
        ipv4_address: 10.0.100.22
    ports:
      - "2181:2181"

  kafka_server:
    build:
      context: kafka/
      dockerfile: Dockerfile
    environment:
      - KAFKA_ACTION=start-kafka
    container_name: kafkaServer
    hostname: kafkaServer
    restart: always
    depends_on:
      - kafka_zk
    networks:
      aqm:
        ipv4_address: 10.0.100.23
    ports:
      - "9092:9092"

  kafka_ui:
    image: provectuslabs/kafka-ui:v0.7.2
    environment:
      - KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=10.0.100.23:9092
      - KAFKA_CLUSTERS_0_ZOOKEEPER=10.0.100.22:2181
    container_name: kafkaUI
    networks:
      aqm:
    ports:
      - "8080:8080"

  logstash:
    image: docker.elastic.co/logstash/logstash:8.13.0
    hostname: logstash
    container_name: logstash
    volumes:
      - ./logstash/pipeline/from_python_to_kafka.conf:/usr/share/logstash/pipeline/logstash.conf
    environment:
      - XPACK_MONITORING_ENABLED=false
    networks:
      aqm:
  
  ingestion_manager:
    build:
      context: ingestion_manager
      dockerfile: Dockerfile
    hostname: ingestion_manager
    container_name: ingestion_manager
    volumes:
      - ./ingestion_manager:/ingestion_manager
    depends_on:
      - logstash
    networks:
      aqm:

  spark:
    build:
      context: spark/
      dockerfile: Dockerfile
    hostname: spark
    container_name: spark
    restart: always
    volumes:
      - ./spark/code/app.py:/opt/aqm/app.py
      - ./ingestion_manager/data/historical_data.csv:/opt/spark/work-dir/data.csv
      - ./spark/model:/opt/spark/work-dir/model
    command: > 
      /opt/spark/bin/spark-submit --conf spark.driver.extraJavaOptions="-Divy.cache.dir=/tmp -Divy.home=/tmp" --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.1,org.elasticsearch:elasticsearch-spark-30_2.12:8.13.4  /opt/aqm/app.py
    networks:
      aqm:
    ports:
      - "4040:4040"
    depends_on:
      elasticsearch:
        condition: service_healthy

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:8.13.4
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
    mem_limit: 1 GB
    hostname: elasticsearch
    container_name: elasticsearch
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9200/_cluster/health"]
      interval: 15s
      timeout: 15s
      retries: 5
      start_period: 90s
    networks:
      aqm:
    ports:
      - "9200:9200"

  kibana:
    hostname: kibana
    container_name: kibana
    image: docker.elastic.co/kibana/kibana:8.13.4
    ports:
      - "5601:5601"
    volumes:
      - ./kibana/kibana.yml:/usr/share/kibana/config/kibana.yml
    networks:
      aqm:


# Uncomment to train the model with historical data
  # train_model:
  #   build:
  #     context: spark/
  #     dockerfile: Dockerfile
  #   volumes:
  #     - ./spark/code/train_model.py:/opt/aqm/train_model.py
  #     - ./ingestion_manager/data/historical_data.csv:/opt/spark/work-dir/data.csv
  #     - ./spark/model:/opt/spark/work-dir/model
  #   command: > 
  #     /opt/spark/bin/spark-submit --conf spark.driver.extraJavaOptions="-Divy.cache.dir=/tmp -Divy.home=/tmp" --packages org.apache.spark:spark-sql-kafka-0-10_2.12:3.5.1 /opt/aqm/train_model.py