From e2a17210ec1011684a31e1602831d24366a636d8 Mon Sep 17 00:00:00 2001 From: Babis Kiosidis Date: Tue, 13 Dec 2022 15:22:00 +0200 Subject: [PATCH 1/3] start and use sandbox --- .circleci/config.yml | 6 +++--- .../java/com/spotify/styx/e2e_tests/EndToEndTestBase.java | 6 ++++-- .../src/test/resources/styx-e2e-test-scheduler.conf | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f8a12940e6..23c67a6211 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -62,9 +62,9 @@ jobs: # Maven heap size MAVEN_OPTS: -Xmx256m -Xms256m command: | - google-cloud-sdk/bin/kubectl -n flyte port-forward deployment/flyteadmin 8089:8089 & - # it takes some time for the portforwarding to be ready - grpc_health/grpc_health_probe -addr 127.0.0.1:8089 -connect-timeout 8s + curl -sL https://ctl.flyte.org/install | bash + flytectl sandbox start --dev + flytectl register examples -d development -p flytesnacks --config ~/.flyte/config-sandbox.yaml --version v0.3.154 source google-cloud-sdk/path.bash.inc mvn -B verify -P disable-static-analysis - run: diff --git a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/EndToEndTestBase.java b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/EndToEndTestBase.java index e672f325d8..4fec389baf 100644 --- a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/EndToEndTestBase.java +++ b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/EndToEndTestBase.java @@ -110,7 +110,7 @@ public class EndToEndTestBase { private static final String FLYTE_DOMAIN = "development"; private static final String FLYTE_LAUNCH_PLAN_NAME = "morning_greeting"; - private static final String FLYTE_LAUNCH_PLAN_VERSION = "04e346ca5f43fc2778259d77a2d9f64ce42b2a27"; + private static final String FLYTE_LAUNCH_PLAN_VERSION = "v0.3.154"; private static final String RESOURCE_TYPE = "LAUNCH_PLAN"; private static final Map REFERENCE_ID = Map.of( @@ -123,7 +123,9 @@ public class EndToEndTestBase { protected static final Map FLYTE_EXEC_CONF_MAP = Map.of( "reference_id", REFERENCE_ID, - "input_fields", Map.of() + "input_fields", Map.of( + "day_of_week", 1 + ) ); private final CompletableFuture styxSchedulerInstance = new CompletableFuture<>(); diff --git a/styx-e2e-test/src/test/resources/styx-e2e-test-scheduler.conf b/styx-e2e-test/src/test/resources/styx-e2e-test-scheduler.conf index aafdecdd2d..df9ddd6a5b 100644 --- a/styx-e2e-test/src/test/resources/styx-e2e-test-scheduler.conf +++ b/styx-e2e-test/src/test/resources/styx-e2e-test-scheduler.conf @@ -11,8 +11,8 @@ styx.gke.default.namespace = ${styx.test.namespace} # flyte conf styx.flyte.enabled = true -styx.flyte.admin.default.host = "127.0.0.1" -styx.flyte.admin.default.port = 8089 +styx.flyte.admin.default.host = "localhost" +styx.flyte.admin.default.port = 30080 styx.flyte.admin.default.insecure = true From 78061aa9e9303f60d5c36d1206761a4a67bbb4fc Mon Sep 17 00:00:00 2001 From: Babis Kiosidis Date: Tue, 13 Dec 2022 15:22:17 +0200 Subject: [PATCH 2/3] run e2e tests --- .../test/java/com/spotify/styx/e2e_tests/AdHocTriggeringIT.java | 2 -- .../src/test/java/com/spotify/styx/e2e_tests/BackfillIT.java | 2 -- .../java/com/spotify/styx/e2e_tests/CreateDeleteWorkflowIT.java | 2 -- .../java/com/spotify/styx/e2e_tests/ScheduledTriggeringIT.java | 2 -- 4 files changed, 8 deletions(-) diff --git a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/AdHocTriggeringIT.java b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/AdHocTriggeringIT.java index 9cb655cff9..826d620dd6 100644 --- a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/AdHocTriggeringIT.java +++ b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/AdHocTriggeringIT.java @@ -33,14 +33,12 @@ import java.util.List; import java.util.Map; -import org.junit.Ignore; import org.junit.Test; @AutoService(EndToEndTestBase.class) public class AdHocTriggeringIT extends EndToEndTestBase { @Test - @Ignore public void testAdhocTriggeringFlyteWorkflow() throws Exception { // Generate workflow configuration var workflowJson = Json.OBJECT_MAPPER.writeValueAsString(Map.of( diff --git a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/BackfillIT.java b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/BackfillIT.java index 35b43f2c45..2a6ab32b6f 100644 --- a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/BackfillIT.java +++ b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/BackfillIT.java @@ -38,7 +38,6 @@ import java.util.Map; import java.util.stream.Stream; -import org.junit.Ignore; import org.junit.Test; @AutoService(EndToEndTestBase.class) @@ -57,7 +56,6 @@ public void testBackfill() throws Exception { } @Test - @Ignore public void testBackfillFlyteWorkflow() throws Exception { // Generate workflow configuration var workflowJson = Json.OBJECT_MAPPER.writeValueAsString(Map.of( diff --git a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/CreateDeleteWorkflowIT.java b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/CreateDeleteWorkflowIT.java index 0ba3545289..d29df57eef 100644 --- a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/CreateDeleteWorkflowIT.java +++ b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/CreateDeleteWorkflowIT.java @@ -36,7 +36,6 @@ import java.util.List; import java.util.Optional; -import org.junit.Ignore; import org.junit.Test; @AutoService(EndToEndTestBase.class) @@ -56,7 +55,6 @@ public void testCreateDeleteWorkflow() throws Exception { } @Test - @Ignore public void testCreateDeleteFlyteWorkflow() throws Exception { var workflowConfiguration = WorkflowConfiguration.builder() .id(workflowId1) diff --git a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/ScheduledTriggeringIT.java b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/ScheduledTriggeringIT.java index 5c0dd6fc88..0963c3f1ae 100644 --- a/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/ScheduledTriggeringIT.java +++ b/styx-e2e-test/src/test/java/com/spotify/styx/e2e_tests/ScheduledTriggeringIT.java @@ -34,14 +34,12 @@ import java.util.List; import java.util.Map; -import org.junit.Ignore; import org.junit.Test; @AutoService(EndToEndTestBase.class) public class ScheduledTriggeringIT extends EndToEndTestBase { @Test - @Ignore public void testScheduledTriggeringFlyteWorkflow() throws Exception { // Generate workflow configuration var workflowJson = From 67e12a046363f08524e13c405f1b35fa398f256f Mon Sep 17 00:00:00 2001 From: Babis Kiosidis Date: Tue, 13 Dec 2022 15:41:41 +0200 Subject: [PATCH 3/3] Fix curl --- .circleci/config.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 23c67a6211..aeb4af8be7 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -54,6 +54,8 @@ jobs: chmod +x grpc_health/grpc_health_probe echo 'dc5c8cd7b33ef503f49eb639af6017f33d340884 grpc_health/grpc_health_probe' > .grpc_checksum sha1sum -c .grpc_checksum + - setup_remote_docker: + version: 20.10.14 - run: name: Tests environment: @@ -62,9 +64,9 @@ jobs: # Maven heap size MAVEN_OPTS: -Xmx256m -Xms256m command: | - curl -sL https://ctl.flyte.org/install | bash - flytectl sandbox start --dev - flytectl register examples -d development -p flytesnacks --config ~/.flyte/config-sandbox.yaml --version v0.3.154 + curl -kL https://ctl.flyte.org/install | bash + ./bin/flytectl sandbox start --dev + ./bin/flytectl register examples -d development -p flytesnacks --config ~/.flyte/config-sandbox.yaml --version v0.3.154 source google-cloud-sdk/path.bash.inc mvn -B verify -P disable-static-analysis - run: