diff --git a/.github/workflows/github-e2e-manual.yaml b/.github/workflows/github-e2e-manual.yaml index 01fa5d663..dcf6941a2 100644 --- a/.github/workflows/github-e2e-manual.yaml +++ b/.github/workflows/github-e2e-manual.yaml @@ -10,6 +10,7 @@ on: options: - scylla - mongo + - rs3 test-type: description: 'Which test variant to run' type: choice diff --git a/.github/workflows/github-e2e.yaml b/.github/workflows/github-e2e.yaml index 63a772d99..017e342c7 100644 --- a/.github/workflows/github-e2e.yaml +++ b/.github/workflows/github-e2e.yaml @@ -102,7 +102,7 @@ jobs: password: ${{ secrets.ANTITHESIS_PASSWORD }} github_token: ${{ secrets.ANTITHESIS_GH_PAT }} config_image: us-central1-docker.pkg.dev/molten-verve-216720/responsive-repository/e2e-test-config-${{ inputs.test-type }}:${{ steps.kafka_client_version.outputs.KAFKA_CLIENT_VERSION }} - images: us-central1-docker.pkg.dev/molten-verve-216720/responsive-repository/e2e-test:${{ steps.kafka_client_version.outputs.KAFKA_CLIENT_VERSION }};docker.io/mongo:5.0 + images: us-central1-docker.pkg.dev/molten-verve-216720/responsive-repository/e2e-test:${{ steps.kafka_client_version.outputs.KAFKA_CLIENT_VERSION }};docker.io/mongo:5.0;public.ecr.aws/j8q9y0n6/responsiveinc/rs3-antithesis:0.1.0-main.30.0 description: "responsive antithesis test run" email_recipients: "antithesis-responsive-aaaamurlsqy6e3hxnx6ksnec5y@antithesisgroup.slack.com" additional_parameters: |- diff --git a/kafka-client-examples/e2e-test/docker/antithesis/async/Dockerfile b/kafka-client-examples/e2e-test/docker/antithesis/async/Dockerfile index ba5f025ae..744a0b175 100644 --- a/kafka-client-examples/e2e-test/docker/antithesis/async/Dockerfile +++ b/kafka-client-examples/e2e-test/docker/antithesis/async/Dockerfile @@ -1,8 +1,9 @@ -from scratch +FROM scratch COPY base/config/docker-compose-base.yml /docker-compose-base.yml COPY base/config/docker-compose-scylla.yml /docker-compose-scylla.yml COPY base/config/docker-compose-mongo.yml /docker-compose-mongo.yml +COPY base/config/docker-compose-rs3.yml /docker-compose-rs3.yml COPY base/config/volumes/scylla.yaml volumes/scylla.yaml ADD base/config/volumes/zookeeper /volumes/zookeeper ADD base/config/volumes/scylla /volumes/scylla @@ -10,6 +11,7 @@ ADD base/config/volumes/scylla2 /volumes/scylla2 ADD base/config/volumes/scylla3 /volumes/scylla3 ADD base/config/volumes/kafka /volumes/kafka +COPY async/config/rs3.toml volumes/rs3.toml COPY async/config/docker-compose.yml /docker-compose.yml COPY async/config/volumes/app.properties /volumes/app.properties COPY async/config/volumes/log4j2.properties /volumes/log4j2.properties diff --git a/kafka-client-examples/e2e-test/docker/antithesis/async/config/rs3.toml b/kafka-client-examples/e2e-test/docker/antithesis/async/config/rs3.toml new file mode 100644 index 000000000..734671e0b --- /dev/null +++ b/kafka-client-examples/e2e-test/docker/antithesis/async/config/rs3.toml @@ -0,0 +1,18 @@ +[server] +listen_port = 50051 + +[store_provider] +provider = "Static" + +[store_provider.static_store.slatedb] +wal_enabled = false + +[store_provider.static_store.slatedb.block_cache] +mode = "Default" + +[[store_provider.static_store.stores]] +id = "b1a45157-e2f0-4698-be0e-5bf3a9b8e9d1" +pss = [{ id = 0 }, { id = 1 }, { id = 2 }, { id = 3 }, { id = 4 }, { id = 5 }, { id = 6 }, { id = 7 }] + +[cloud_provider] +provider = "InMemory" diff --git a/kafka-client-examples/e2e-test/docker/antithesis/async/config/volumes/app-rs3.properties b/kafka-client-examples/e2e-test/docker/antithesis/async/config/volumes/app-rs3.properties new file mode 100644 index 000000000..030bf8cf0 --- /dev/null +++ b/kafka-client-examples/e2e-test/docker/antithesis/async/config/volumes/app-rs3.properties @@ -0,0 +1,21 @@ +# +# Copyright 2024 Responsive Computing, Inc. +# +# This source code is licensed under the Responsive Business Source License Agreement v1.0 +# available at: +# +# https://www.responsive.dev/legal/responsive-bsl-10 +# +# This software requires a valid Commercial License Key for production use. Trial and commercial +# licenses can be obtained at https://www.responsive.dev +# + +bootstrap.servers=broker:9092 +responsive.org=responsive +responsive.env=test + +responsive.storage.backend.type=RS3 +responsive.rs3.hostname=rs3 +responsive.rs3.port=50051 +responsive.rs3.logical.store.mapping=e2e:b1a45157-e2f0-4698-be0e-5bf3a9b8e9d1 +responsive.rs3.tls.enabled=false diff --git a/kafka-client-examples/e2e-test/docker/antithesis/base/config/docker-compose-rs3.yml b/kafka-client-examples/e2e-test/docker/antithesis/base/config/docker-compose-rs3.yml new file mode 100644 index 000000000..e3fd35a62 --- /dev/null +++ b/kafka-client-examples/e2e-test/docker/antithesis/base/config/docker-compose-rs3.yml @@ -0,0 +1,18 @@ +include: + - docker-compose-base.yml +services: + rs3: + # Remember to update the image in github-e2e.yaml as well + image: public.ecr.aws/j8q9y0n6/responsiveinc/rs3-antithesis:0.1.0-main.30.0 + platform: linux/amd64 + container_name: rs3 + hostname: rs3 + ports: + - 50051:50051 + environment: + CONFIG_FILE: /etc/rs3/rs3.toml + volumes: + - ./volumes/rs3.toml:/etc/rs3/rs3.toml + networks: + backend: + ipv4_address: 10.0.0.60 diff --git a/kafka-client-examples/e2e-test/docker/antithesis/regression/Dockerfile b/kafka-client-examples/e2e-test/docker/antithesis/regression/Dockerfile index 746b6832e..45cbfeb89 100644 --- a/kafka-client-examples/e2e-test/docker/antithesis/regression/Dockerfile +++ b/kafka-client-examples/e2e-test/docker/antithesis/regression/Dockerfile @@ -1,8 +1,9 @@ -from scratch +FROM scratch COPY base/config/docker-compose-base.yml /docker-compose-base.yml COPY base/config/docker-compose-scylla.yml /docker-compose-scylla.yml COPY base/config/docker-compose-mongo.yml /docker-compose-mongo.yml +COPY base/config/docker-compose-rs3.yml /docker-compose-rs3.yml COPY base/config/volumes/scylla.yaml volumes/scylla.yaml ADD base/config/volumes/zookeeper /volumes/zookeeper ADD base/config/volumes/scylla /volumes/scylla @@ -13,6 +14,7 @@ ADD base/config/volumes/mongo2 /volumes/mongo2 ADD base/config/volumes/mongo3 /volumes/mongo3 ADD base/config/volumes/kafka /volumes/kafka +COPY regression/config/rs3.toml volumes/rs3.toml COPY regression/config/docker-compose.yml /docker-compose.yml COPY regression/config/volumes/app.properties /volumes/app.properties COPY regression/config/volumes/log4j2.properties /volumes/log4j2.properties diff --git a/kafka-client-examples/e2e-test/docker/antithesis/regression/config/rs3.toml b/kafka-client-examples/e2e-test/docker/antithesis/regression/config/rs3.toml new file mode 100644 index 000000000..57ab59f58 --- /dev/null +++ b/kafka-client-examples/e2e-test/docker/antithesis/regression/config/rs3.toml @@ -0,0 +1,22 @@ +[server] +listen_port = 50051 + +[store_provider] +provider = "Static" + +[store_provider.static_store.slatedb] +wal_enabled = false + +[store_provider.static_store.slatedb.block_cache] +mode = "Default" + +[[store_provider.static_store.stores]] +id = "b1a45157-e2f0-4698-be0e-5bf3a9b8e9d1" +pss = [{ id = 0 }, { id = 1 }, { id = 2 }, { id = 3 }, { id = 4 }, { id = 5 }, { id = 6 }, { id = 7 }] + +[[store_provider.static_store.stores]] +id = "013a57c3-16b3-44be-8de4-0d1083671cc6" +pss = [{ id = 0 }, { id = 1 }, { id = 2 }, { id = 3 }, { id = 4 }, { id = 5 }, { id = 6 }, { id = 7 }] + +[cloud_provider] +provider = "InMemory" diff --git a/kafka-client-examples/e2e-test/docker/antithesis/regression/config/volumes/app-rs3.properties b/kafka-client-examples/e2e-test/docker/antithesis/regression/config/volumes/app-rs3.properties new file mode 100644 index 000000000..0f7f39515 --- /dev/null +++ b/kafka-client-examples/e2e-test/docker/antithesis/regression/config/volumes/app-rs3.properties @@ -0,0 +1,21 @@ +# +# Copyright 2024 Responsive Computing, Inc. +# +# This source code is licensed under the Responsive Business Source License Agreement v1.0 +# available at: +# +# https://www.responsive.dev/legal/responsive-bsl-10 +# +# This software requires a valid Commercial License Key for production use. Trial and commercial +# licenses can be obtained at https://www.responsive.dev +# + +bootstrap.servers=broker:9092 +responsive.org=responsive +responsive.env=test + +responsive.storage.backend.type=RS3 +responsive.rs3.hostname=rs3 +responsive.rs3.port=50051 +responsive.rs3.logical.store.mapping=customers_state_store_0000000001:b1a45157-e2f0-4698-be0e-5bf3a9b8e9d1,KSTREAM-REDUCE-STATE-STORE-0000000006:013a57c3-16b3-44be-8de4-0d1083671cc6 +responsive.rs3.tls.enabled=false diff --git a/kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/e2etest/E2ETestApplication.java b/kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/e2etest/E2ETestApplication.java index 314430c63..cec6a86f4 100644 --- a/kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/e2etest/E2ETestApplication.java +++ b/kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/e2etest/E2ETestApplication.java @@ -34,6 +34,7 @@ import org.apache.kafka.streams.KafkaStreams; import org.apache.kafka.streams.StreamsBuilder; import org.apache.kafka.streams.StreamsConfig; +import org.apache.kafka.streams.Topology; import org.apache.kafka.streams.kstream.Consumed; import org.apache.kafka.streams.kstream.KStream; import org.apache.kafka.streams.kstream.Named; @@ -140,10 +141,11 @@ private KafkaStreams buildTopology(final Map properties) { builderProperties.put(StreamsConfig.PROCESSING_GUARANTEE_CONFIG, StreamsConfig.EXACTLY_ONCE_V2); builderProperties.put(ResponsiveConfig.ASYNC_THREAD_POOL_SIZE_CONFIG, 2); - final var streams = new ResponsiveKafkaStreams( - builder.build(builderProperties), - builderProperties - ); + + Topology topology = builder.build(builderProperties); + LOG.info("Starting Kafka Streams with topology \n{}", topology.describe()); + + final var streams = new ResponsiveKafkaStreams(topology, builderProperties); streams.setUncaughtExceptionHandler(new UncaughtStreamsAntithesisHandler()); return streams; } diff --git a/kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/regression/tests/AbstractKSExampleService.java b/kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/regression/tests/AbstractKSExampleService.java index e518de1fe..22ccafe64 100644 --- a/kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/regression/tests/AbstractKSExampleService.java +++ b/kafka-client-examples/e2e-test/src/main/java/dev/responsive/examples/regression/tests/AbstractKSExampleService.java @@ -82,7 +82,9 @@ protected final void startUp() throws Exception { E2ETestUtils.awaitKeyspace(properties); LOG.info("Created keyspace..."); - LOG.info("Starting Kafka Streams..."); + Topology topology = buildTopology(); + LOG.info("Starting Kafka Streams with topology \n{}", topology.describe()); + kafkaStreams = responsive ? new ResponsiveKafkaStreams(buildTopology(), properties) : new KafkaStreams(buildTopology(), new StreamsConfig(properties));