diff --git a/.env b/.env new file mode 100644 index 000000000..6a4ff6e52 --- /dev/null +++ b/.env @@ -0,0 +1,4 @@ +ELK_VERSION=6.5.4 +RABBITMQ_VERSION=3.7.9 +POSTGRES_VERSION=11.2-alpine +REDIS_VERSION=5.0.4 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 000000000..b2c9f5b4b --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,138 @@ +version: '3' + +services: + + + ###### EPP ####### + epp_db: + image: "postgres:$POSTGRES_VERSION" + ports: + - 3003:5432 + environment: + POSTGRES_DB: epp + POSTGRES_USER: arkiv + POSTGRES_PASSWORD: password + + epp_rabbitmq: + image: "rabbitmq:$RABBITMQ_VERSION" + ports: + - 15003:5672 + environment: + - RABBITMQ_DEFAULT_USER=rabbitmq + - RABBITMQ_DEFAULT_PASS=rabbitmq + + epp_redis: + image: "redis:$REDIS_VERSION" + ports: + - 6003:6379 + + + ###### ETA ####### + eta_db: + image: "postgres:$POSTGRES_VERSION" + ports: + - 3002:5432 + environment: + POSTGRES_DB: eta + POSTGRES_USER: arkiv + POSTGRES_PASSWORD: password + + eta_rabbitmq: + image: "rabbitmq:$RABBITMQ_VERSION" + ports: + - 15002:5672 + environment: + - RABBITMQ_DEFAULT_USER=rabbitmq + - RABBITMQ_DEFAULT_PASS=rabbitmq + + eta_redis: + image: "redis:$REDIS_VERSION" + ports: + - 6002:6379 + + + ###### ETP ####### + etp_db: + image: "postgres:$POSTGRES_VERSION" + ports: + - 3001:5432 + environment: + POSTGRES_DB: etp + POSTGRES_USER: arkiv + POSTGRES_PASSWORD: password + + etp_rabbitmq: + image: "rabbitmq:$RABBITMQ_VERSION" + ports: + - 15001:5672 + environment: + - RABBITMQ_DEFAULT_USER=rabbitmq + - RABBITMQ_DEFAULT_PASS=rabbitmq + + etp_redis: + image: "redis:$REDIS_VERSION" + ports: + - 6001:6379 + + + ###### ELK Stack ####### + elasticsearch: + build: + context: docker/elasticsearch + args: + ELK_VERSION: $ELK_VERSION + volumes: + - ./docker/elasticsearch/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml:ro + ports: + - "9200:9200" + - "9300:9300" + - "9202:9200" + - "9302:9300" + - "9203:9200" + - "9303:9300" + environment: + ES_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + + logstash: + build: + context: docker/logstash + args: + ELK_VERSION: $ELK_VERSION + volumes: + - ./docker/logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro + - ./docker/logstash/pipeline:/usr/share/logstash/pipeline:ro + ports: + - "5000:5000" + - "9600:9600" + - "5002:5000" + - "9602:9600" + - "5003:5000" + - "9603:9600" + environment: + LS_JAVA_OPTS: "-Xmx256m -Xms256m" + networks: + - elk + depends_on: + - elasticsearch + + kibana: + build: + context: docker/kibana + args: + ELK_VERSION: $ELK_VERSION + volumes: + - ./docker/kibana/config/:/usr/share/kibana/config:ro + ports: + - "5601:5601" + - "5602:5601" + - "5603:5601" + networks: + - elk + depends_on: + - elasticsearch + +networks: + elk: + driver: bridge diff --git a/docker/elasticsearch/Dockerfile b/docker/elasticsearch/Dockerfile new file mode 100644 index 000000000..1f4604ed6 --- /dev/null +++ b/docker/elasticsearch/Dockerfile @@ -0,0 +1,8 @@ +ARG ELK_VERSION + +# https://github.com/elastic/elasticsearch-docker +FROM docker.elastic.co/elasticsearch/elasticsearch:${ELK_VERSION} + +# Add your elasticsearch plugins setup here + +RUN elasticsearch-plugin install --batch ingest-attachment diff --git a/docker/elasticsearch/config/elasticsearch.yml b/docker/elasticsearch/config/elasticsearch.yml new file mode 100644 index 000000000..e97577084 --- /dev/null +++ b/docker/elasticsearch/config/elasticsearch.yml @@ -0,0 +1,16 @@ +--- +## Default Elasticsearch configuration from elasticsearch-docker. +## from https://github.com/elastic/elasticsearch-docker/blob/master/build/elasticsearch/elasticsearch.yml +# +cluster.name: "docker-cluster" +network.host: 0.0.0.0 + +# minimum_master_nodes need to be explicitly set when bound on a public IP +# set to 1 to allow single node clusters +# Details: https://github.com/elastic/elasticsearch/pull/17288 +discovery.zen.minimum_master_nodes: 1 + +## Use single node discovery in order to disable production mode and avoid bootstrap checks +## see https://www.elastic.co/guide/en/elasticsearch/reference/current/bootstrap-checks.html +# +discovery.type: single-node diff --git a/docker/kibana/Dockerfile b/docker/kibana/Dockerfile new file mode 100644 index 000000000..202855cc9 --- /dev/null +++ b/docker/kibana/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://github.com/elastic/kibana-docker +FROM docker.elastic.co/kibana/kibana:${ELK_VERSION} + +# Add your kibana plugins setup here +# Example: RUN kibana-plugin install diff --git a/docker/kibana/config/kibana.yml b/docker/kibana/config/kibana.yml new file mode 100644 index 000000000..4b34a1606 --- /dev/null +++ b/docker/kibana/config/kibana.yml @@ -0,0 +1,7 @@ +--- +## Default Kibana configuration from kibana-docker. +## from https://github.com/elastic/kibana-docker/blob/master/build/kibana/config/kibana.yml +# +server.name: kibana +server.host: "0" +elasticsearch.url: http://elasticsearch:9200 diff --git a/docker/logstash/Dockerfile b/docker/logstash/Dockerfile new file mode 100644 index 000000000..6cc863745 --- /dev/null +++ b/docker/logstash/Dockerfile @@ -0,0 +1,7 @@ +ARG ELK_VERSION + +# https://github.com/elastic/logstash-docker +FROM docker.elastic.co/logstash/logstash:${ELK_VERSION} + +# Add your logstash plugins setup here +# Example: RUN logstash-plugin install logstash-filter-json diff --git a/docker/logstash/config/logstash.yml b/docker/logstash/config/logstash.yml new file mode 100644 index 000000000..e7e260f22 --- /dev/null +++ b/docker/logstash/config/logstash.yml @@ -0,0 +1,6 @@ +--- +## Default Logstash configuration from logstash-docker. +## from https://github.com/elastic/logstash-docker/blob/master/build/logstash/config/logstash-oss.yml +# +http.host: "0.0.0.0" +path.config: /usr/share/logstash/pipeline diff --git a/docker/logstash/pipeline/logstash.conf b/docker/logstash/pipeline/logstash.conf new file mode 100644 index 000000000..875320e7c --- /dev/null +++ b/docker/logstash/pipeline/logstash.conf @@ -0,0 +1,29 @@ +input { + tcp { + port => 5000 + codec => json + } +} + +## Add your filters / logstash plugins configuration here +filter { + json { + source => "[message][raw]" + } + + # Workaround for HTTP logs created from "django.channels.server" that have an extra "\u001b[m" at start and "\u001b[0m" at end. + if [type] == "django_http" { + grok { + match => { "message" => "m%{URIPROTO:protocol} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:status_code} \[%{NUMBER:duration}, %{HOSTPORT:host}\]" } + } + mutate { + remove_field => [ "message" ] + } + } +} + +output { + elasticsearch { + hosts => "elasticsearch:9200" + } +}