Skip to content

Create multi-broker-kafka-without-zookeeper #163

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open

Create multi-broker-kafka-without-zookeeper #163

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
263 changes: 263 additions & 0 deletions multi-broker-kafka-without-zookeeper
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,263 @@
# This docker-compose is only to get started with Conduktor.
# It is starting Conduktor, a Redpanda cluster (3 brokers), and a fake app to generate traffic.

# Go to http://localhost:8080 when started

####################################################################################################
# DO NOT USE IT IN PRODUCTION
#
# For production, please check: https://docs.conduktor.io/platform/category/deployment-options/
# to deploy on Kubernetes via Helm, or AWS using Cloudformation.
####################################################################################################


services:
# Conduktor Console, the enterprise UI.
# It depends on PostgreSQL and connects to our Redpanda cluster.
# https://docs.conduktor.io/platform/get-started/configuration/introduction/
conduktor-console:
image: conduktor/conduktor-console:1.34.0
ports:
- "8080:8080"
volumes:
- conduktor_data:/var/conduktor
environment:
CDK_DATABASE_URL: "postgresql://conduktor:change_me@postgresql:5432/conduktor-console"
CDK_KAFKASQL_DATABASE_URL: "postgresql://conduktor:change_me@postgresql-sql:5432/conduktor-sql"
CDK_ORGANIZATION_NAME: "getting-started"
CDK_CLUSTERS_0_ID: "local-kafka"
CDK_CLUSTERS_0_NAME: "local-kafka"
CDK_CLUSTERS_0_BOOTSTRAPSERVERS: "redpanda-1:9092,redpanda-2:9092,redpanda-3:9092"
CDK_CLUSTERS_0_SCHEMAREGISTRY_URL: "http://redpanda-1:8081"
CDK_CLUSTERS_0_COLOR: "#6A57C8"
CDK_CLUSTERS_0_ICON: "kafka"
CDK_CLUSTERS_1_ID: "cdk-gateway"
CDK_CLUSTERS_1_NAME: "cdk-gateway"
CDK_CLUSTERS_1_BOOTSTRAPSERVERS: "conduktor-gateway:6969"
CDK_CLUSTERS_1_SCHEMAREGISTRY_URL: "http://redpanda-1:8081"
CDK_CLUSTERS_1_KAFKAFLAVOR_URL: "http://conduktor-gateway:8888"
CDK_CLUSTERS_1_KAFKAFLAVOR_USER: "admin"
CDK_CLUSTERS_1_KAFKAFLAVOR_PASSWORD: "conduktor"
CDK_CLUSTERS_1_KAFKAFLAVOR_VIRTUALCLUSTER: "passthrough"
CDK_CLUSTERS_1_KAFKAFLAVOR_TYPE: "Gateway"
CDK_CLUSTERS_1_COLOR: "#6A57C8"
CDK_CLUSTERS_1_ICON: "dog"
CDK_MONITORING_CORTEX-URL: http://conduktor-monitoring:9009/
CDK_MONITORING_ALERT-MANAGER-URL: http://conduktor-monitoring:9010/
CDK_MONITORING_CALLBACK-URL: http://conduktor-console:8080/monitoring/api/
CDK_MONITORING_NOTIFICATIONS-CALLBACK-URL: http://localhost:8080
depends_on:
redpanda-1:
condition: service_healthy
redpanda-2:
condition: service_healthy
redpanda-3:
condition: service_healthy
postgresql:
condition: service_healthy
postgresql-2:
condition: service_healthy

# Conduktor stores its metadata in PostgreSQL.
# Consider using an external managed database for production usage.
# https://docs.conduktor.io/platform/get-started/configuration/database/
postgresql:
image: postgres:14
hostname: postgresql
volumes:
- pg_data:/var/lib/postgresql/data
environment:
PGDATA: "/var/lib/postgresql/data"
POSTGRES_DB: "conduktor-console"
POSTGRES_USER: "conduktor"
POSTGRES_PASSWORD: "change_me"
POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
healthcheck:
test: ["CMD-SHELL", "pg_isready"]
interval: 10s
timeout: 5s
retries: 5

# Conduktor depends on a separate db for storing Kafka data for SQL querying
# It is optional, but required if you wish to use SQL functionality
# Separate db ensures continued operation of the core Console experience if the SQL db becomes unavailable
# https://docs.conduktor.io/platform/guides/configure-sql/
postgresql-2:
image: postgres:14
hostname: postgresql-sql
volumes:
- pg_data_sql:/var/lib/postgresql/data
environment:
PGDATA: "/var/lib/postgresql/data"
POSTGRES_DB: "conduktor-sql"
POSTGRES_USER: "conduktor"
POSTGRES_PASSWORD: "change_me"
POSTGRES_HOST_AUTH_METHOD: "scram-sha-256"
healthcheck:
test: ["CMD-SHELL", "pg_isready"]
interval: 10s
timeout: 5s
retries: 5

# Conduktor uses Cortex to store Kafka and applications metrics as well as alerting.
# It is optional.
# https://docs.conduktor.io/platform/get-started/configuration/cortex/
conduktor-monitoring:
image: conduktor/conduktor-console-cortex:1.34.0
environment:
CDK_CONSOLE-URL: "http://conduktor-console:8080"

# Redpanda broker 1 (leader)
redpanda-1:
image: docker.redpanda.com/redpandadata/redpanda:v24.1.6
hostname: redpanda-1
container_name: redpanda-1
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:19092
- --advertise-kafka-addr internal://redpanda-1:9092,external://localhost:19092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:18082
- --advertise-pandaproxy-addr internal://redpanda-1:8082,external://localhost:18082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:18081
- --rpc-addr redpanda-1:33145
- --advertise-rpc-addr redpanda-1:33145
- --node-id 0
- --smp 1
- --memory 1G
- --mode dev-container
- --default-log-level=info
volumes:
- redpanda-1:/var/lib/redpanda/data
ports:
- 18081:18081
- 18082:18082
- 19092:19092
- 19644:9644
healthcheck:
test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
interval: 15s
timeout: 3s
retries: 5
start_period: 5s

# Redpanda broker 2
redpanda-2:
image: docker.redpanda.com/redpandadata/redpanda:v24.1.6
hostname: redpanda-2
container_name: redpanda-2
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:29092
- --advertise-kafka-addr internal://redpanda-2:9092,external://localhost:29092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:28082
- --advertise-pandaproxy-addr internal://redpanda-2:8082,external://localhost:28082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:28081
- --rpc-addr redpanda-2:33145
- --advertise-rpc-addr redpanda-2:33145
- --seeds redpanda-1:33145
- --node-id 1
- --smp 1
- --memory 1G
- --mode dev-container
- --default-log-level=info
volumes:
- redpanda-2:/var/lib/redpanda/data
ports:
- 28081:28081
- 28082:28082
- 29092:29092
- 29644:9644
depends_on:
redpanda-1:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
interval: 15s
timeout: 3s
retries: 5
start_period: 5s

# Redpanda broker 3
redpanda-3:
image: docker.redpanda.com/redpandadata/redpanda:v24.1.6
hostname: redpanda-3
container_name: redpanda-3
command:
- redpanda
- start
- --kafka-addr internal://0.0.0.0:9092,external://0.0.0.0:39092
- --advertise-kafka-addr internal://redpanda-3:9092,external://localhost:39092
- --pandaproxy-addr internal://0.0.0.0:8082,external://0.0.0.0:38082
- --advertise-pandaproxy-addr internal://redpanda-3:8082,external://localhost:38082
- --schema-registry-addr internal://0.0.0.0:8081,external://0.0.0.0:38081
- --rpc-addr redpanda-3:33145
- --advertise-rpc-addr redpanda-3:33145
- --seeds redpanda-1:33145
- --node-id 2
- --smp 1
- --memory 1G
- --mode dev-container
- --default-log-level=info
volumes:
- redpanda-3:/var/lib/redpanda/data
ports:
- 38081:38081
- 38082:38082
- 39092:39092
- 39644:9644
depends_on:
redpanda-1:
condition: service_healthy
healthcheck:
test: ["CMD-SHELL", "rpk cluster health | grep -E 'Healthy:.+true' || exit 1"]
interval: 15s
timeout: 3s
retries: 5
start_period: 5s

# Conduktor comes with its Gateway, a Kafka proxy bringing many security and governance features.
# Now Gateway uses our Redpanda cluster as its backend.
# https://docs.conduktor.io/gateway/
conduktor-gateway:
image: conduktor/conduktor-gateway:3.9.0
hostname: conduktor-gateway
container_name: conduktor-gateway
environment:
KAFKA_BOOTSTRAP_SERVERS: redpanda-1:9092,redpanda-2:9092,redpanda-3:9092
ports:
- "8888:8888"
healthcheck:
test: curl localhost:8888/health
interval: 5s
retries: 25
depends_on:
redpanda-1:
condition: service_healthy
redpanda-2:
condition: service_healthy
redpanda-3:
condition: service_healthy

# As this is a get started, we want to bring some life to the cluster to demonstrate the value of Conduktor.
# This is totally optional and only used for this purpose. Do not use it in production.
conduktor-data-generator:
image: conduktor/conduktor-data-generator:0.9
container_name: conduktor-data-generator
environment:
KAFKA_BOOTSTRAP_SERVERS: conduktor-gateway:6969
KAFKA_SCHEMA_REGISTRY_URL: http://redpanda-1:8081
GATEWAY_ADMIN_API: http://conduktor-gateway:8888
restart: on-failure
depends_on:
conduktor-gateway:
condition: service_healthy

volumes:
pg_data: {}
pg_data_sql: {}
conduktor_data: {}
redpanda-1: {}
redpanda-2: {}
redpanda-3: {}