From b0e9a24738930778441c647260d3384f1b4512bc Mon Sep 17 00:00:00 2001 From: niels Date: Wed, 14 Nov 2018 10:33:30 +0100 Subject: [PATCH 1/2] StorageTapper demo using docker-compose --- docker/Dockerfile | 20 +++++++++++ docker/Dockerfile_demo | 10 ++++++ docker/Dockerfile_kafkaconsumer | 7 ++++ docker/README.md | 4 +++ docker/config/config-file.cnf | 8 +++++ docker/config/production.yaml | 8 +++++ docker/demo.sh | 48 ++++++++++++++++++++++++++ docker/docker-compose.yml | 61 +++++++++++++++++++++++++++++++++ docker/wait_for.sh | 15 ++++++++ 9 files changed, 181 insertions(+) create mode 100644 docker/Dockerfile create mode 100644 docker/Dockerfile_demo create mode 100644 docker/Dockerfile_kafkaconsumer create mode 100644 docker/README.md create mode 100644 docker/config/config-file.cnf create mode 100644 docker/config/production.yaml create mode 100644 docker/demo.sh create mode 100644 docker/docker-compose.yml create mode 100644 docker/wait_for.sh diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 0000000..549b5e6 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,20 @@ +FROM golang:stretch as build + +RUN go get github.com/Masterminds/glide && \ + go get github.com/alecthomas/gometalinter && \ + go get github.com/tinylib/msgp + +RUN apt-get update && apt-get install -y netcat + +RUN mkdir -p $GOPATH/src/github.com/uber && \ + cd $GOPATH/src/github.com/uber && \ + git clone https://github.com/uber/storagetapper.git && \ + cd storagetapper && \ + rm -f pipe/hdfs.go && \ + DEB_BUILD_OPTIONS=nocheck make install + +ADD wait_for.sh /wait_for.sh +RUN chmod +x /wait_for.sh + +VOLUME ['/etc/storagetapper/'] +ENTRYPOINT ['/wait_for.sh', 'storagetapper'] \ No newline at end of file diff --git a/docker/Dockerfile_demo b/docker/Dockerfile_demo new file mode 100644 index 0000000..a054e98 --- /dev/null +++ b/docker/Dockerfile_demo @@ -0,0 +1,10 @@ +FROM debian:stretch + +RUN apt-get update && apt-get install -y netcat mysql-client curl + +ADD wait_for.sh /wait_for.sh +ADD demo.sh /demo.sh +RUN chmod +x /wait_for.sh /demo.sh + +ENTRYPOINT ["/wait_for.sh", "/demo.sh"] +CMD [""] \ No newline at end of file diff --git a/docker/Dockerfile_kafkaconsumer b/docker/Dockerfile_kafkaconsumer new file mode 100644 index 0000000..77b6567 --- /dev/null +++ b/docker/Dockerfile_kafkaconsumer @@ -0,0 +1,7 @@ +FROM wurstmeister/kafka + +ADD wait_for.sh /wait_for.sh +RUN chmod +x /wait_for.sh + +ENTRYPOINT ["/wait_for.sh", "/opt/kafka/bin/kafka-console-consumer.sh"] +CMD [''] \ No newline at end of file diff --git a/docker/README.md b/docker/README.md new file mode 100644 index 0000000..c8a2aed --- /dev/null +++ b/docker/README.md @@ -0,0 +1,4 @@ +Docker Demo +----------- + +A docker-compose highlighting how to run storagetapper and capture snapshot/changes for an mysql table \ No newline at end of file diff --git a/docker/config/config-file.cnf b/docker/config/config-file.cnf new file mode 100644 index 0000000..3846086 --- /dev/null +++ b/docker/config/config-file.cnf @@ -0,0 +1,8 @@ +[mysqld] +server-id=1 +binlog-format=ROW +gtid_mode=ON +enforce-gtid-consistency +log_slave_updates=1 +log-bin=bin.log +log-bin-index=bin-log.index \ No newline at end of file diff --git a/docker/config/production.yaml b/docker/config/production.yaml new file mode 100644 index 0000000..6db7bdc --- /dev/null +++ b/docker/config/production.yaml @@ -0,0 +1,8 @@ +log_type: std +state_update_timeout: 10 + +state_connect_url: root:storagetapper@db +kafka_addresses: + - "kafka:9092" +changelog_topic_name_template_default: "storagetapper.changelog.{{.Service}}.{{.Db}}.{{.Table}}" +output_topic_name_template_default: "storagetapper.output.{{.Service}}.{{.Db}}.{{.Table}}" \ No newline at end of file diff --git a/docker/demo.sh b/docker/demo.sh new file mode 100644 index 0000000..b56f16f --- /dev/null +++ b/docker/demo.sh @@ -0,0 +1,48 @@ +#!/bin/bash + +set -e + +sql() { + db=$1 + shift + mysql -uroot -pstoragetapper -hdb "$db" -e "$@" +} + + +sql "" "DROP DATABASE IF EXISTS ex_db1" +sql "" "RESET MASTER" + +sql "" "CREATE DATABASE ex_db1" +sql "ex_db1" "CREATE TABLE ex_table1(id int not null primary key, ts TIMESTAMP, type varchar(32))" + +for i in `seq 1 10`; do + sql "ex_db1" "INSERT INTO ex_table1(id, type) VALUES ($i, 'pre-existing')" +done + + +curl -s --data '{"cmd" : "add", "name" : "ex_cluster1", "host" : "db", "port" : 3306, "user" : "root", "pw" : "storagetapper"}' http://storagetapper:7836/cluster +curl -s --data '{"cmd" : "add", "cluster" : "ex_cluster1", "service" : "ex_svc1", "db":"ex_db1", "table":"ex_table1", "output":"kafka", "outputFormat":"json"}' http://storagetapper:7836/table + + +sleep 12 + +for j in `seq 1 100`; do + +let start=$j*100 +let end=$start+10 + +for i in `seq $start $end`; do + sql "ex_db1" "INSERT INTO ex_table1(id, type) VALUES ($i, 'incremental')" + sleep 1 +done + +sql "ex_db1" "ALTER TABLE ex_table1 ADD schema_change varchar(32)" + +for i in `seq $start $end`; do + sql "ex_db1" "UPDATE ex_table1 SET type = 'update' WHERE id = $i;" + sleep 1 +done + +sql "ex_db1" "ALTER TABLE ex_table1 DROP schema_change" + +done \ No newline at end of file diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml new file mode 100644 index 0000000..c2adbed --- /dev/null +++ b/docker/docker-compose.yml @@ -0,0 +1,61 @@ +version: '3.1' + +services: + + db: + image: mysql:5 + command: --default-authentication-plugin=mysql_native_password + restart: always + environment: + # Use root/storagetapper as user/password credentials + MYSQL_ROOT_PASSWORD: storagetapper + volumes: + - ./config:/etc/mysql/conf.d/ + security_opt: + - seccomp:unconfined + + adminer: + image: adminer + restart: always + ports: + - 8080:8080 + + zookeeper: + image: wurstmeister/zookeeper + + kafka: + image: wurstmeister/kafka + environment: + KAFKA_CREATE_TOPICS: "storagetapper.changelog.ex_svc1.ex_db1.ex_table1:1:1,storagetapper.output.ex_svc1.ex_db1.ex_table1:1:1" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_ADVERTISED_HOST_NAME: kafka + + kafka_client_changelog: + build: + context: . + dockerfile: Dockerfile_kafkaconsumer + environment: + - WAIT_FOR=kafka:9092 + command: --bootstrap-server kafka:9092 --topic storagetapper.changelog.ex_svc1.ex_db1.ex_table1 --from-beginning + + kafka_client_output: + build: + context: . + dockerfile: Dockerfile_kafkaconsumer + environment: + - WAIT_FOR=kafka:9092 + command: --bootstrap-server kafka:9092 --topic storagetapper.output.ex_svc1.ex_db1.ex_table1 --from-beginning + + storagetapper: + build: . + environment: + - WAIT_FOR=db:3306;kafka:9092 + volumes: + - ./config:/etc/storagetapper/ + + demo: + build: + context: . + dockerfile: Dockerfile_demo + environment: + - WAIT_FOR=storagetapper:7836 diff --git a/docker/wait_for.sh b/docker/wait_for.sh new file mode 100644 index 0000000..c2e2cba --- /dev/null +++ b/docker/wait_for.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +if [ -n "$WAIT_FOR" ]; then + IFS=';' read -a HOSTPORT_ARRAY <<< "$WAIT_FOR" + for HOSTPORT in "${HOSTPORT_ARRAY[@]}" + do + WAIT_FOR_HOST=${HOSTPORT%:*} + WAIT_FOR_PORT=${HOSTPORT#*:} + + echo Waiting for $WAIT_FOR_HOST to listen on $WAIT_FOR_PORT... + while ! nc -z $WAIT_FOR_HOST $WAIT_FOR_PORT; do echo sleeping; sleep 2; done + done +fi + +exec "$@" \ No newline at end of file From 67b331fc88ed157949bc56f1445dcfaf522e3ebb Mon Sep 17 00:00:00 2001 From: niels Date: Wed, 14 Nov 2018 10:36:50 +0100 Subject: [PATCH 2/2] Fixup workflow example, REST API changed --- scripts/workflow_example.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/workflow_example.sh b/scripts/workflow_example.sh index 7fee073..dfbd7e9 100644 --- a/scripts/workflow_example.sh +++ b/scripts/workflow_example.sh @@ -35,7 +35,7 @@ trap "kill $TPID; exit 1" SIGINT SIGTERM SIGHUP sleep 2 curl --data '{"cmd" : "add", "name" : "ex_cluster1", "host" : "localhost", "port" : 3306, "user" : "root", "pw" : ""}' http://localhost:7836/cluster -curl --data '{"cmd" : "add", "cluster" : "ex_cluster1", "service" : "ex_svc1", "db":"ex_db1", "table":"ex_table1"}' http://localhost:7836/table +curl --data '{"cmd" : "add", "cluster" : "ex_cluster1", "service" : "ex_svc1", "db":"ex_db1", "table":"ex_table1", "output":"kafka", "outputFormat":"json"}' http://localhost:7836/table sleep 12