docker-compose.yaml

version: '3.8'

services:



  #-------------------------------#
  # 🖥 logstash                   #
  #-------------------------------#

  logstash:
    build:
      context: logstash
      dockerfile: Dockerfile
    image: logstash
    volumes:
      - ./logstash/pipeline/:/usr/share/logstash/pipeline/
    environment:
      - "LS_JAVA_OPTS=-Xms1g -Xmx1g"
      - "xpack.monitoring.enabled:true"
      - "xpack.monitoring.elasticsearch.url:http://localhost:9200"
    ports:
      - "10155:10155"
    networks:
      - energy-network

  logstash2:
    build:
      context: logstash
      dockerfile: Dockerfile
    image: logstash
    volumes:
      - ./logstash/pipeline/:/usr/share/logstash/pipeline/
    environment:
      - "LS_JAVA_OPTS=-Xms1g -Xmx1g"
      - "xpack.monitoring.enabled:true"
      - "xpack.monitoring.elasticsearch.url:http://localhost:9200"
    ports:
      - "10156:10155"
    networks:
      - energy-network


  #-------------------------------#
  # 🖥 Data-server                #
  #-------------------------------#


  data-server:
    container_name: data-server-container
    build:
      context: server
      dockerfile: Dockerfile
    image: data-server
    ports:
      - "8000:8000"
    networks:
      - energy-network


  
  #-------------------------------#
  # 🖥 Zookeeper                      #
  #-------------------------------#

  zookeeper:
    image: confluentinc/cp-zookeeper:7.0.1
    container_name: zookeeper
    hostname: zookeeper
    ports:
      - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000
    networks:
      - energy-network

  #-------------------------------#
  # 🖥 Kafka                      #
  #-------------------------------#
  kafkaserver:
    image: confluentinc/cp-kafka:7.0.1
    container_name: kafkaserver
    hostname: kafkaserver
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_INTERNAL:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafkaserver:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
    networks:
      - energy-network

  #-------------------------------#
  # 🖥 Kafka UI                   #
  #-------------------------------#
  kafka-ui:
    image: provectuslabs/kafka-ui
    container_name: kafka-ui
    depends_on:
      - zookeeper
      - kafkaserver
    ports:
      - "8080:8080"
    restart: always
    environment:
      KAFKA_CLUSTERS_0_NAME: local
      KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS: kafkaserver:9092
      KAFKA_CLUSTERS_0_ZOOKEEPER: zookeeper:2181
    networks:
      - energy-network

  #-------------------------------#
  # 🖥 Init Kafka                 #
  #-------------------------------#
  init-kafka:
    image: confluentinc/cp-kafka:6.1.1
    networks:
      - energy-network
    depends_on:
      - kafkaserver
      - zookeeper
      - kafka-ui
    entrypoint: [ '/bin/sh', '-c' ]
    command: |
      "
      # blocks until kafka is reachable
      kafka-topics --bootstrap-server kafkaserver:9092 --list

      echo -e 'Creating kafka topics'
      kafka-topics --bootstrap-server kafkaserver:9092 --create --if-not-exists --topic dati_energetici --replication-factor 1 --partitions 1

      echo -e 'Successfully created the following topics:'
      kafka-topics --bootstrap-server kafkaserver:9092 --list
      "

  #-------------------------------#
  # 🖥 Spark Streaming            #
  #-------------------------------#
  # spark:
  #   build: 
  #     context: Spark
  #   container_name: sparkcontainer
  #   depends_on: 
  #     - kafkaserver
  #     - logstash
  #     - elasticsearch
  #   restart: on-failure
  #   networks: 
  #     - energy-network
  #   deploy:
  #     resources:
  #       limits:
  #         cpus: '4.0'
  #         memory: 4g


  spark:
    build:
      context: Spark
    container_name: sparkcontainer
    networks:
      - energy-network
    volumes:
      - certs:/app/certs
    depends_on:
      zookeeper:
        condition: service_started
      kafkaserver:
        condition: service_started
      setup:
        condition: service_completed_successfully
      elasticsearch01:
        condition: service_started
      kibana:
        condition: service_started
    deploy:
      resources:
        limits:
          cpus: '4.0'
          memory: 4g

  #-------------------------------#
  # 🖥SETUP ES e KIBANA             #
  #-------------------------------#

  setup:
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    container_name: setupESKibana 
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
    user: "0"
    command: >
      bash -c '
        if [ x${ELASTIC_PASSWORD} == x ]; then
          echo "Set the ELASTIC_PASSWORD environment variable in the .env file";
          exit 1;
        elif [ x${KIBANA_PASSWORD} == x ]; then
          echo "Set the KIBANA_PASSWORD environment variable in the .env file";
          exit 1;
        fi;
        if [ ! -f config/certs/ca.zip ]; then
          echo "Creating CA";
          bin/elasticsearch-certutil ca --silent --pem -out config/certs/ca.zip;
          unzip config/certs/ca.zip -d config/certs;
        fi;
        if [ ! -f config/certs/certs.zip ]; then
          echo "Creating certs";
          echo -ne \
          "instances:\n"\
          "  - name: elasticsearch01\n"\
          "    dns:\n"\
          "      - elasticsearch01\n"\
          "      - localhost\n"\
          "    ip:\n"\
          "      - 127.0.0.1\n"\
          > config/certs/instances.yml;
          bin/elasticsearch-certutil cert --silent --pem -out config/certs/certs.zip --in config/certs/instances.yml --ca-cert config/certs/ca/ca.crt --ca-key config/certs/ca/ca.key;
          unzip config/certs/certs.zip -d config/certs;
        fi;
        echo "Setting file permissions"
        chown -R root:root config/certs;
        find . -type d -exec chmod 750 \{\} \;;
        find . -type f -exec chmod 640 \{\} \;;
        echo "Waiting for Elasticsearch availability";
        until curl -s --cacert config/certs/ca/ca.crt https://elasticsearch01:9200 | grep -q "missing authentication credentials"; do sleep 30; done;
        echo "Setting kibana_system password";
        until curl -s -X POST --cacert config/certs/ca/ca.crt -u elastic:${ELASTIC_PASSWORD} -H "Content-Type: application/json" https://elasticsearch01:9200/_security/user/kibana_system/_password -d "{\"password\":\"${KIBANA_PASSWORD}\"}" | grep -q "^{}"; do sleep 10; done;
        echo "All done!";
        chmod 777 -R config/certs
      '
    healthcheck:
      test: ["CMD-SHELL", "[ -f config/certs/elasticsearch01/elasticsearch01.crt ]"]
      interval: 1s
      timeout: 5s
      retries: 120
    networks: 
      - energy-network

  #-------------------------------#
  # 🖥 Elasticsearch              #
  #-------------------------------#
  # elasticsearch:
  #   container_name: elasticsearch

  #   image: docker.elastic.co/elasticsearch/elasticsearch:8.2.0
  #   ports:
  #     - "9200:9200"

  #   environment:
  #     - node.name=elasticsearch
  #     - xpack.security.enabled=false
  #     - discovery.type=single-node
  #     - bootstrap.memory_lock=true
  #     - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
  #     - cluster.routing.allocation.disk.threshold_enabled=false
  #   ulimits:
  #     memlock:
  #       soft: -1
  #       hard: -1
  #   networks: 
  #   - energy-network


  elasticsearch01:
    depends_on:
      setup:
        condition: service_healthy
    image: docker.elastic.co/elasticsearch/elasticsearch:${STACK_VERSION}
    hostname: elasticsearch01
    container_name: elasticsearch01 
    volumes:
      - certs:/usr/share/elasticsearch/config/certs
      - esdata01:/usr/share/elasticsearch/data
    ports:
      - ${ES_PORT}:9200
    environment:
      - node.name=elasticsearch01
      - cluster.name=${CLUSTER_NAME}
      - cluster.initial_master_nodes=elasticsearch01
      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
      - bootstrap.memory_lock=true
      - xpack.security.enabled=true
      - xpack.security.http.ssl.enabled=true
      - xpack.security.http.ssl.key=certs/elasticsearch01/elasticsearch01.key
      - xpack.security.http.ssl.certificate=certs/elasticsearch01/elasticsearch01.crt
      - xpack.security.http.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.http.ssl.verification_mode=certificate
      - xpack.security.transport.ssl.enabled=true
      - xpack.security.transport.ssl.key=certs/elasticsearch01/elasticsearch01.key
      - xpack.security.transport.ssl.certificate=certs/elasticsearch01/elasticsearch01.crt
      - xpack.security.transport.ssl.certificate_authorities=certs/ca/ca.crt
      - xpack.security.transport.ssl.verification_mode=certificate
      - xpack.license.self_generated.type=${LICENSE}
      - "ES_JAVA_OPTS=-Xms3g -Xmx3g"
    ulimits:
      memlock:
        soft: -1
        hard: -1
    healthcheck:
      test: ["CMD-SHELL","curl -s --cacert config/certs/ca/ca.crt https://localhost:9200 | grep -q 'missing authentication credentials'",]
      interval: 10s
      timeout: 10s
      retries: 120
    networks: 
      - energy-network


  #-------------------------------#
  # 🖥 Kibana                     #
  #-------------------------------#


  # kibana:
  #   container_name: kibana-container
  #   build:
  #       context: Kibana    
  #       dockerfile: Dockerfile
  #   image: kibana            
  #   hostname: kibana
  #   ports:
  #     - "5601:5601"
  #   environment:
  #     - xpack.security.enabled=false
  #   depends_on: 
  #     - elasticsearch
  #   networks: 
  #     - energy-network


  kibana:
    build:
      context: Kibana    
      dockerfile: Dockerfile
    depends_on:
      elasticsearch01:
        condition: service_healthy
    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
    container_name: kibana 
    volumes:
      - certs:/usr/share/kibana/config/certs
      - kibanadata:/usr/share/kibana/data
      - ./Kibana/elasticsearch/configs/kibana.yml:/usr/share/kibana/config/kibana.yml
    ports:
      - ${KIBANA_PORT}:5601
    environment:
      - SERVERNAME=kibana
      - ELASTICSEARCH_HOSTS=https://elasticsearch01:9200
      - ELASTICSEARCH_USERNAME=kibana_system
      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
      - ELASTICSEARCH_SSL_CERTIFICATEAUTHORITIES=config/certs/ca/ca.crt
    mem_limit: ${MEM_LIMIT}
    healthcheck:
      test: ["CMD-SHELL","curl -s -I http://localhost:5601 | grep -q 'HTTP/1.1 302 Found'",]
      interval: 20s
      timeout:  20s
      retries: 120
    networks: 
      - energy-network

networks:
  energy-network:
    name: energy-network
    driver: bridge

volumes:
  certs:
    driver: local
  esdata01:
    driver: local
  kibanadata:
    driver: local