From 4ab065a5f1d340ce160d55245e2e3e42eff4d857 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 11 Oct 2024 22:08:41 +0800 Subject: [PATCH 001/104] tmp store --- .../dolphinscheduler/alert/AlertServer.java | 4 ++- .../alert/config/AlertConfig.java | 3 ++ .../src/main/resources/application.yaml | 5 +++ .../api/ApiApplicationServer.java | 4 +++ .../src/main/resources/application.yaml | 5 +++ .../base/client/NettyRemotingClient.java | 22 ++++++++++++ .../base/config/NettyClientConfig.java | 3 ++ .../base/config/NettyServerConfig.java | 5 +++ .../extract/base/config/NettySslConfig.java | 34 +++++++++++++++++++ .../base/server/NettyRemotingServer.java | 24 +++++++++++-- .../server/master/MasterServer.java | 7 +++- .../server/master/config/MasterConfig.java | 3 ++ .../src/main/resources/application.yaml | 5 +++ .../server/worker/WorkerServer.java | 6 +++- .../server/worker/config/WorkerConfig.java | 3 +- .../src/main/resources/application.yaml | 5 +++ 16 files changed, 132 insertions(+), 6 deletions(-) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java index ff2e98542629..6e3688e6cc75 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/AlertServer.java @@ -25,6 +25,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.registry.api.RegistryConfiguration; import javax.annotation.PostConstruct; @@ -40,7 +41,8 @@ @Slf4j @Import({CommonConfiguration.class, DaoConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class AlertServer { diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java index 240f92b846f8..084bc34c48fb 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java @@ -26,6 +26,7 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import org.springframework.validation.Errors; @@ -47,6 +48,8 @@ public final class AlertConfig implements Validator { private String alertServerAddress; + private NettySslConfig nettySslConfig; + @Override public boolean supports(Class clazz) { return AlertConfig.class.isAssignableFrom(clazz); diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml index 927cbc3c2ce6..7251d9becb18 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml @@ -75,6 +75,11 @@ alert: max-heartbeat-interval: 60s # The maximum number of alerts that can be processed in parallel sender-parallelism: 100 + rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem registry: type: zookeeper diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index b072fe630878..c4c2acce3459 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.dao.DaoConfiguration; import org.apache.dolphinscheduler.dao.PluginDao; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; import org.apache.dolphinscheduler.plugin.storage.api.StorageConfiguration; import org.apache.dolphinscheduler.plugin.task.api.TaskPluginManager; @@ -52,6 +53,9 @@ public class ApiApplicationServer { @Autowired private PluginDao pluginDao; + @Autowired + NettySslConfig nettySslConfig; + public static void main(String[] args) { ApiServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); Thread.setDefaultUncaughtExceptionHandler(DefaultUncaughtExceptionHandler.getInstance()); diff --git a/dolphinscheduler-api/src/main/resources/application.yaml b/dolphinscheduler-api/src/main/resources/application.yaml index 9b0e94d64451..8ce78518c571 100644 --- a/dolphinscheduler-api/src/main/resources/application.yaml +++ b/dolphinscheduler-api/src/main/resources/application.yaml @@ -164,6 +164,11 @@ api: # Close each active connection of socket server if python program not active after x milliseconds. Define value is # (0 = infinite), and socket server would never close even though no requests accept read-timeout: 0 + rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem metrics: enabled: true diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java index 4aea4d6dfe9c..fed57160d239 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/client/NettyRemotingClient.java @@ -21,6 +21,7 @@ import org.apache.dolphinscheduler.extract.base.IRpcResponse; import org.apache.dolphinscheduler.extract.base.SyncRequestDto; import org.apache.dolphinscheduler.extract.base.config.NettyClientConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.exception.RemotingException; import org.apache.dolphinscheduler.extract.base.exception.RemotingTimeoutException; import org.apache.dolphinscheduler.extract.base.future.ResponseFuture; @@ -33,6 +34,7 @@ import org.apache.dolphinscheduler.extract.base.utils.Host; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; +import java.io.File; import java.net.InetSocketAddress; import java.util.Map; import java.util.concurrent.ConcurrentHashMap; @@ -41,6 +43,8 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.locks.ReentrantLock; +import javax.net.ssl.SSLException; + import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.Bootstrap; import io.netty.channel.Channel; @@ -52,6 +56,8 @@ import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.timeout.IdleStateHandler; @Slf4j @@ -70,8 +76,20 @@ public class NettyRemotingClient implements AutoCloseable { private final NettyClientHandler clientHandler; + private SslContext sslContext = null; + public NettyRemotingClient(final NettyClientConfig clientConfig) { this.clientConfig = clientConfig; + NettySslConfig nettySslConfig = clientConfig.getNettySslConfig(); + if (nettySslConfig.isEnabled()) { + try { + sslContext = + SslContextBuilder.forClient().trustManager(new File(nettySslConfig.getCertFilePath())).build(); + } catch (SSLException e) { + throw new IllegalArgumentException("Initialize SslContext error, please check the cert-file", e); + } + } + ThreadFactory nettyClientThreadFactory = ThreadUtils.newDaemonThreadFactory("NettyClientThread-"); if (Epoll.isAvailable()) { this.workerGroup = new EpollEventLoopGroup(clientConfig.getWorkerThreads(), nettyClientThreadFactory); @@ -97,6 +115,10 @@ private void start() { @Override public void initChannel(SocketChannel ch) { + NettySslConfig nettySslConfig = clientConfig.getNettySslConfig(); + if (nettySslConfig.isEnabled()) { + ch.pipeline().addLast(sslContext.newHandler(ch.alloc())); + } ch.pipeline() .addLast("client-idle-handler", new IdleStateHandler( diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java index a00ff540f4a5..974f9c43468c 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyClientConfig.java @@ -76,4 +76,7 @@ public class NettyClientConfig { @Builder.Default private int defaultRpcTimeoutMillis = 10_000; + @Builder.Default + private NettySslConfig nettySslConfig = new NettySslConfig(); + } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java index cc0aa04f68c9..d7051dc37a47 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java @@ -79,4 +79,9 @@ public class NettyServerConfig { */ private int listenPort; + /** + * nettySslConfig + */ + @Builder.Default + private NettySslConfig nettySslConfig = new NettySslConfig(); } diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java new file mode 100644 index 000000000000..2524712fb3a3 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettySslConfig.java @@ -0,0 +1,34 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.config; + +import lombok.Data; + +import org.springframework.context.annotation.Configuration; + +@Configuration +@Data +public class NettySslConfig { + + public boolean enabled; + + public String certFilePath; + + public String keyFilePath; + +} diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java index 9ebf802b1ed2..f9c010170e75 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/server/NettyRemotingServer.java @@ -19,16 +19,20 @@ import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.extract.base.config.NettyServerConfig; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.extract.base.exception.RemoteException; import org.apache.dolphinscheduler.extract.base.protocal.TransporterDecoder; import org.apache.dolphinscheduler.extract.base.protocal.TransporterEncoder; import org.apache.dolphinscheduler.extract.base.utils.NettyUtils; +import java.io.File; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadFactory; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import javax.net.ssl.SSLException; + import lombok.Getter; import lombok.extern.slf4j.Slf4j; import io.netty.bootstrap.ServerBootstrap; @@ -40,8 +44,9 @@ import io.netty.channel.epoll.EpollEventLoopGroup; import io.netty.channel.nio.NioEventLoopGroup; import io.netty.channel.socket.SocketChannel; +import io.netty.handler.ssl.SslContext; +import io.netty.handler.ssl.SslContextBuilder; import io.netty.handler.timeout.IdleStateHandler; - /** * remoting netty server */ @@ -66,7 +71,18 @@ class NettyRemotingServer { private final AtomicBoolean isStarted = new AtomicBoolean(false); - NettyRemotingServer(final NettyServerConfig serverConfig) { + private SslContext sslContext = null; + + public NettyRemotingServer(final NettyServerConfig serverConfig) { + NettySslConfig nettySslConfig = serverConfig.getNettySslConfig(); + if (nettySslConfig.isEnabled()) { + try { + sslContext = SslContextBuilder.forServer(new File(nettySslConfig.getCertFilePath()), + new File(nettySslConfig.getKeyFilePath())).build(); + } catch (SSLException e) { + throw new RuntimeException(e); + } + } this.serverConfig = serverConfig; this.serverName = serverConfig.getServerName(); this.methodInvokerExecutor = ThreadUtils.newDaemonFixedThreadExecutor( @@ -130,6 +146,10 @@ protected void initChannel(SocketChannel ch) { * @param ch socket channel */ private void initNettyChannel(SocketChannel ch) { + NettySslConfig nettySslConfig = serverConfig.getNettySslConfig(); + if (nettySslConfig.isEnabled()) { + ch.pipeline().addLast("ssl", sslContext.newHandler(ch.alloc())); + } ch.pipeline() .addLast("encoder", new TransporterEncoder()) .addLast("decoder", new TransporterDecoder()) diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 4cffd3e37224..953efba7c6fc 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -24,6 +24,7 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -60,7 +61,8 @@ ServiceConfiguration.class, CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class MasterServer implements IStoppable { @@ -94,6 +96,9 @@ public class MasterServer implements IStoppable { @Autowired private SystemEventBusFireWorker systemEventBusFireWorker; + @Autowired + NettySslConfig nettySslConfig; + public static void main(String[] args) { MasterServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java index 9ff59e0cdb94..8c79367c28f8 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/config/MasterConfig.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.master.config; import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; import org.apache.dolphinscheduler.server.master.cluster.loadbalancer.WorkerLoadBalancerConfigurationProperties; @@ -79,6 +80,8 @@ public class MasterConfig implements Validator { */ private String masterRegistryPath; + private NettySslConfig nettySslConfig; + @Override public boolean supports(Class clazz) { return MasterConfig.class.isAssignableFrom(clazz); diff --git a/dolphinscheduler-master/src/main/resources/application.yaml b/dolphinscheduler-master/src/main/resources/application.yaml index 964fca836ce6..dc19ce8e4297 100644 --- a/dolphinscheduler-master/src/main/resources/application.yaml +++ b/dolphinscheduler-master/src/main/resources/application.yaml @@ -121,6 +121,11 @@ master: memory-usage-weight: 30 cpu-usage-weight: 30 task-thread-pool-usage-weight: 40 + rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem server: port: 5679 diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index 4618c5ad59b6..f833967de630 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -23,6 +23,7 @@ import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -55,7 +56,8 @@ @Slf4j @Import({CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @SpringBootApplication public class WorkerServer implements IStoppable { @@ -71,6 +73,8 @@ public class WorkerServer implements IStoppable { @Autowired private MetricsProvider metricsProvider; + @Autowired + NettySslConfig nettySslConfig; /** * worker server startup, not use web service * diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java index d901effbd1db..7841d6d4eaea 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/config/WorkerConfig.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.server.worker.config; import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.registry.api.ConnectStrategyProperties; import org.apache.dolphinscheduler.registry.api.enums.RegistryNodeType; @@ -47,7 +48,7 @@ public class WorkerConfig implements Validator { private int hostWeight = 100; private WorkerServerLoadProtection serverLoadProtection = new WorkerServerLoadProtection(); private ConnectStrategyProperties registryDisconnectStrategy = new ConnectStrategyProperties(); - + private NettySslConfig nettySslConfig; /** * This field doesn't need to set at config file, it will be calculated by workerIp:listenPort */ diff --git a/dolphinscheduler-worker/src/main/resources/application.yaml b/dolphinscheduler-worker/src/main/resources/application.yaml index 5cac4c29e5ae..4bc55ce1ed87 100644 --- a/dolphinscheduler-worker/src/main/resources/application.yaml +++ b/dolphinscheduler-worker/src/main/resources/application.yaml @@ -68,6 +68,11 @@ worker: auto-create-tenant-enabled: true # If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. default-tenant-enabled: false + rpc: + ssl: + enabled: false + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem server: port: 1235 From 9d0515bb8e4fd231a54a8cd0f56a83f4e373d03c Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 12 Oct 2024 09:20:08 +0800 Subject: [PATCH 002/104] format the code --- .../org/apache/dolphinscheduler/alert/config/AlertConfig.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java index 084bc34c48fb..a5217ed9d510 100644 --- a/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java +++ b/dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/java/org/apache/dolphinscheduler/alert/config/AlertConfig.java @@ -18,6 +18,7 @@ package org.apache.dolphinscheduler.alert.config; import org.apache.dolphinscheduler.common.utils.NetUtils; +import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.commons.lang3.StringUtils; @@ -26,7 +27,6 @@ import lombok.Data; import lombok.extern.slf4j.Slf4j; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.stereotype.Component; import org.springframework.validation.Errors; From 71a167a75ddcd7d143a1baa0d78f1438524f263e Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 12 Oct 2024 14:36:15 +0800 Subject: [PATCH 003/104] fix ut --- .../org/apache/dolphinscheduler/api/ApiApplicationServer.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java index c4c2acce3459..8341a1cbb59f 100644 --- a/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java +++ b/dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/ApiApplicationServer.java @@ -45,7 +45,8 @@ CommonConfiguration.class, ServiceConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class}) + RegistryConfiguration.class, + NettySslConfig.class}) @ServletComponentScan @SpringBootApplication public class ApiApplicationServer { From d3dc34f59056c7b93ecb83b2638c7bfdc11988dd Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 16 Oct 2024 15:37:11 +0800 Subject: [PATCH 004/104] add e2e gluster-test --- .../docker/gluster-test/api/application.yaml | 254 ++++++++++++++++++ .../docker/gluster-test/docker-compose.yml | 166 ++++++++++++ .../gluster-test/master/application.yaml | 168 ++++++++++++ .../docker/gluster-test/ssl/cert.crt | 41 +++ .../docker/gluster-test/ssl/private.pem | 45 ++++ .../gluster-test/worker/application.yaml | 99 +++++++ 6 files changed, 773 insertions(+) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml new file mode 100644 index 000000000000..8305faa6fd3c --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml @@ -0,0 +1,254 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +server: + port: 12345 + servlet: + session: + timeout: 120m + context-path: /dolphinscheduler/ + compression: + enabled: true + mime-types: text/html,text/xml,text/plain,text/css,text/javascript,application/javascript,application/json,application/xml + jetty: + max-http-form-post-size: 5000000 + accesslog: + enabled: true + custom-format: '%{client}a - %u %t "%r" %s %O %{ms}Tms' + +spring: + profiles: + active: postgresql + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + servlet: + multipart: + max-file-size: 1024MB + max-request-size: 1024MB + messages: + basename: i18n/messages + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + quartz: + auto-startup: false + job-store-type: jdbc + jdbc: + initialize-schema: never + properties: + org.quartz.jobStore.isClustered: true + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore + org.quartz.scheduler.instanceId: AUTO + org.quartz.jobStore.tablePrefix: QRTZ_ + org.quartz.jobStore.acquireTriggersWithinLock: true + org.quartz.scheduler.instanceName: DolphinScheduler + org.quartz.threadPool.class: org.apache.dolphinscheduler.scheduler.quartz.QuartzZeroSizeThreadPool + org.quartz.jobStore.useProperties: false + org.quartz.jobStore.misfireThreshold: 60000 + org.quartz.scheduler.makeSchedulerThreadDaemon: true + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.clusterCheckinInterval: 5000 + org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1 + mvc: + pathmatch: + matching-strategy: ANT_PATH_MATCHER + static-path-pattern: /static/** + cloud.discovery.client.composite-indicator.enabled: false + +springdoc: + swagger-ui: + path: /swagger-ui.html + packages-to-scan: org.apache.dolphinscheduler.api + +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 1s + max-sleep: 3s + max-retries: 5 + session-timeout: 60s + connection-timeout: 15s + block-until-connected: 15s + digest: ~ + +api: + audit-enable: false + # Traffic control, if you turn on this config, the maximum number of request/s will be limited. + # global max request number per second + # default tenant-level max request number + traffic-control: + global-switch: false + max-global-qps-rate: 300 + tenant-switch: false + default-tenant-qps-rate: 10 + #customize-tenant-qps-rate: + # eg. + #tenant1: 11 + #tenant2: 20 + python-gateway: + # Weather enable python gateway server or not. The default value is false. + enabled: false + # Authentication token for connection from python api to python gateway server. Should be changed the default value + # when you deploy in public network. + auth-token: jwUDzpLsNKEFER4*a8gruBH_GsAurNxU7A@Xc + # The address of Python gateway server start. Set its value to `0.0.0.0` if your Python API run in different + # between Python gateway server. It could be be specific to other address like `127.0.0.1` or `localhost` + gateway-server-address: 0.0.0.0 + # The port of Python gateway server start. Define which port you could connect to Python gateway server from + # Python API side. + gateway-server-port: 25333 + # The address of Python callback client. + python-address: 127.0.0.1 + # The port of Python callback client. + python-port: 25334 + # Close connection of socket server if no other request accept after x milliseconds. Define value is (0 = infinite), + # and socket server would never close even though no requests accept + connect-timeout: 0 + # Close each active connection of socket server if python program not active after x milliseconds. Define value is + # (0 = infinite), and socket server would never close even though no requests accept + read-timeout: 0 + rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem + +metrics: + enabled: true + +security: + authentication: + # Authentication types (supported types: PASSWORD,LDAP,CASDOOR_SSO) + type: PASSWORD + # IF you set type `LDAP`, below config will be effective + ldap: + # ldap server config + urls: ldap://ldap.forumsys.com:389/ + base-dn: dc=example,dc=com + username: cn=read-only-admin,dc=example,dc=com + password: password + user: + # admin userId when you use LDAP login + admin: read-only-admin + identity-attribute: uid + email-attribute: mail + # action when ldap user is not exist (supported types: CREATE,DENY) + not-exist-action: CREATE + ssl: + enable: false + # jks file absolute path && password + trust-store: "/ldapkeystore.jks" + trust-store-password: "password" + casdoor: + user: + admin: "" + oauth2: + enable: false + provider: + github: + authorizationUri: "" + redirectUri: "" + clientId: "" + clientSecret: "" + tokenUri: "" + userInfoUri: "" + callbackUrl: "" + iconUri: "" + provider: github + google: + authorizationUri: "" + redirectUri: "" + clientId: "" + clientSecret: "" + tokenUri: "" + userInfoUri: "" + callbackUrl: "" + iconUri: "" + provider: google +casdoor: + # Your Casdoor server url + endpoint: "" + client-id: "" + client-secret: "" + # The certificate may be multi-line, you can use `|-` for ease + certificate: "" + # Your organization name added in Casdoor + organization-name: "" + # Your application name added in Casdoor + application-name: "" + # Doplhinscheduler login url + redirect-url: "" + + +# Override by profile + +--- +spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root + quartz: + properties: + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml new file mode 100644 index 000000000000..ada5261f20b1 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml @@ -0,0 +1,166 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +services: + dolphinscheduler-postgresql: + image: bitnami/postgresql:15.2.0 + ports: + - "5432:5432" + profiles: ["all", "schema"] + environment: + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + volumes: + - dolphinscheduler-postgresql:/bitnami/postgresql + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - dolphinscheduler + + dolphinscheduler-zookeeper: + image: bitnami/zookeeper:3.7.1 + profiles: ["all"] + environment: + ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - dolphinscheduler + + dolphinscheduler-schema-initializer: + image: ${HUB}/dolphinscheduler-tools:${TAG} + env_file: .env + profiles: ["schema"] + command: [ tools/bin/upgrade-schema.sh ] + depends_on: + dolphinscheduler-postgresql: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + networks: + - dolphinscheduler + + dolphinscheduler-api: + image: ${HUB}/dolphinscheduler-api:${TAG} + ports: + - "12345:12345" + - "25333:25333" + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + - ./api/application.yaml:/opt/dolphinscheduler/api/conf/application.yaml + - ./ssl/cert.crt:/path/cert.crt + - ./ssl/private.pm:/path/private.pm + networks: + - dolphinscheduler + + dolphinscheduler-alert: + image: ${HUB}/dolphinscheduler-alert-server:${TAG} + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + networks: + - dolphinscheduler + + dolphinscheduler-master: + image: ${HUB}/dolphinscheduler-master:${TAG} + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - ./master/application.yaml:/opt/dolphinscheduler/master/conf/application.yaml + - ./ssl/cert.crt:/path/cert.crt + - ./ssl/private.pm:/path/private.pm + networks: + - dolphinscheduler + + dolphinscheduler-worker: + image: ${HUB}/dolphinscheduler-worker:${TAG} + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + - ./worker/application.yaml:/:/opt/dolphinscheduler/worker/conf/application.yaml + - ./ssl/cert.crt:/path/cert.crt + - ./ssl/private.pm:/path/private.pm + networks: + - dolphinscheduler + +networks: + dolphinscheduler: + driver: bridge + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-worker-data: + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml new file mode 100644 index 000000000000..979d54a02221 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml @@ -0,0 +1,168 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +spring: + profiles: + active: postgresql + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + datasource: + driver-class-name: org.postgresql.Driver + url: jdbc:postgresql://127.0.0.1:5432/dolphinscheduler + username: root + password: root + hikari: + connection-test-query: select 1 + pool-name: DolphinScheduler + quartz: + job-store-type: jdbc + jdbc: + initialize-schema: never + properties: + org.quartz.threadPool.threadPriority: 5 + org.quartz.jobStore.isClustered: true + org.quartz.jobStore.class: org.springframework.scheduling.quartz.LocalDataSourceJobStore + org.quartz.scheduler.instanceId: AUTO + org.quartz.jobStore.tablePrefix: QRTZ_ + org.quartz.jobStore.acquireTriggersWithinLock: true + org.quartz.scheduler.instanceName: DolphinScheduler + org.quartz.threadPool.class: org.quartz.simpl.SimpleThreadPool + org.quartz.jobStore.useProperties: false + org.quartz.threadPool.makeThreadsDaemons: true + org.quartz.threadPool.threadCount: 25 + org.quartz.jobStore.misfireThreshold: 60000 + org.quartz.scheduler.batchTriggerAcquisitionMaxCount: 1 + org.quartz.scheduler.makeSchedulerThreadDaemon: true + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate + org.quartz.jobStore.clusterCheckinInterval: 5000 + cloud.discovery.client.composite-indicator.enabled: false + +# Mybatis-plus configuration, you don't need to change it +mybatis-plus: + mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml + type-aliases-package: org.apache.dolphinscheduler.dao.entity + configuration: + cache-enabled: false + call-setters-on-nulls: true + map-underscore-to-camel-case: true + jdbc-type-for-null: NULL + global-config: + db-config: + id-type: auto + banner: false + + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 1s + max-sleep: 3s + max-retries: 5 + session-timeout: 60s + connection-timeout: 15s + block-until-connected: 15s + digest: ~ + +master: + listen-port: 5678 + # The number of workflow event bus fire worker used to fire the workflow event, default is 2*CPU core + 1. + # workflow-event-bus-fire-thread-count: 10 + # The number of threads used to execute sync logic task e.g. Switch/Condition, default is CPU core. + # master-sync-task-executor-thread-pool-size: 10 + # The number of threads used to execute async logic task e.g. Dependent/SubWorkflow default is CPU core. + # master-async-task-executor-thread-pool-size: 10 + max-heartbeat-interval: 10s + server-load-protection: + # If set true, will open master overload protection + enabled: true + # Master max system cpu usage, when the master's system cpu usage is smaller then this value, master server can execute workflow. + max-system-cpu-usage-percentage-thresholds: 0.7 + # Master max jvm cpu usage, when the master's jvm cpu usage is smaller then this value, master server can execute workflow. + max-jvm-cpu-usage-percentage-thresholds: 0.7 + # Master max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. + max-system-memory-usage-percentage-thresholds: 0.7 + # Master max disk usage , when the master's disk usage is smaller then this value, master server can execute workflow. + max-disk-usage-percentage-thresholds: 0.7 + registry-disconnect-strategy: + # The disconnect strategy: stop, waiting + strategy: stop + worker-group-refresh-interval: 10s + command-fetch-strategy: + type: ID_SLOT_BASED + config: + # The incremental id step + id-step: 1 + # master fetch command num + fetch-size: 10 + worker-load-balancer-configuration-properties: + # RANDOM, ROUND_ROBIN, FIXED_WEIGHTED_ROUND_ROBIN, DYNAMIC_WEIGHTED_ROUND_ROBIN + type: DYNAMIC_WEIGHTED_ROUND_ROBIN + # dynamic-weight-config-properties only used in DYNAMIC_WEIGHTED_ROUND_ROBIN, the weight of memory-usage, cpu-usage, task-thread-pool-usage should sum to 100. + dynamic-weight-config-properties: + memory-usage-weight: 30 + cpu-usage-weight: 30 + task-thread-pool-usage-weight: 40 + rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem + +server: + port: 5679 + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +metrics: + enabled: true + +# Override by profile + +--- +spring: + config: + activate: + on-profile: mysql + datasource: + driver-class-name: com.mysql.cj.jdbc.Driver + url: jdbc:mysql://127.0.0.1:3306/dolphinscheduler + username: root + password: root + quartz: + properties: + org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.StdJDBCDelegate diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt new file mode 100644 index 000000000000..8b26de2debbf --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt @@ -0,0 +1,41 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-----BEGIN CERTIFICATE----- +MIIEDTCCAvUCFCst8gcQmOfuqY8vJ9c4MaacVx+mMA0GCSqGSIb3DQEBCwUAMIHC +MQswCQYDVQQGEwJDTjEQMA4GA1UECAwHYmVpamluZzEQMA4GA1UEBwwHYmVpamlu +ZzEgMB4GA1UECgwXYXBhY2hlIGRvbHBoaW5zY2hlZHVsZXIxFjAUBgNVBAsMDUlU +IERlcGFydG1lbnQxGTAXBgNVBAMMEGRvbHBoaW5zY2hlZHVsZXIxOjA4BgkqhkiG +9w0BCQEWK3VzZXJzLXN1YnNjcmliZUBkb2xwaGluc2NoZWR1bGVyLmFwYWNoZS5v +cmcwHhcNMjQxMDE2MDcyODQ0WhcNMjUxMDE2MDcyODQ0WjCBwjELMAkGA1UEBhMC +Q04xEDAOBgNVBAgMB2JlaWppbmcxEDAOBgNVBAcMB2JlaWppbmcxIDAeBgNVBAoM +F2FwYWNoZSBkb2xwaGluc2NoZWR1bGVyMRYwFAYDVQQLDA1JVCBEZXBhcnRtZW50 +MRkwFwYDVQQDDBBkb2xwaGluc2NoZWR1bGVyMTowOAYJKoZIhvcNAQkBFit1c2Vy +cy1zdWJzY3JpYmVAZG9scGhpbnNjaGVkdWxlci5hcGFjaGUub3JnMIIBIjANBgkq +hkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwr5KUI0ZzVcpOjazaZUDo+y+uEnt1qRC +cmnnNFyMQuVhX5gBHRk+As5e+E+M3TTLpA4LFti644NqN9eUmuqTzEUAfRzd2dfh +cEKFU3TlhpmsLQ7nN/gYe0KMmh9imV7WFV25NPUsiGgEKB5EEFQkq0Ii6A7GHDUD +Cmwcchkwbgr4wCUVJfAjzfSsq1WuNnGlCeIzv9Xv/drqQ2StWuoDav2YoPsMqcTU +yXnrWWUW/L7TLMHZSe7uNphul3BRXNx6c7J9k9QRTDuyJt0seRM7PgFK/9aVKXxo +BG+k2pGLw+Xoq4URNS09YXsBiqEDy5A+RBUivfyE6nGFblraa6I++wIDAQABMA0G +CSqGSIb3DQEBCwUAA4IBAQA7LHY1apCsncI+GcDqvfClWBqJLyCCzLnQ4Z+vLsvC +C0zB0MXpqjXXe/FXcU62DNIXr0IyOlZTApuM7sLZftVYAihGzdMxQuhhoa1yZPo4 +2O6RulbRpxkFBNyfvOAGdmRi+pEfdXX2o/lqzgMqR2vzG79aOUYbgwPXIis4hZOP +Zp1Asz2JojmIRXXYVz00upCIuPs31D/0wZim6RG9RNqToiFCFc+wKENN/fhVwaSC +mz3GKWnagQA7JR6Dz9dVqqPtkhOaSW6KajFU7rvhg/C0VH8i2devOgAW6/7X6e2x ++XHg8jQ6GaYqDdqFk4T9Zpa/aMHBrnB0aHrrTJVrPSei +-----END CERTIFICATE----- diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem new file mode 100644 index 000000000000..a28335c31a91 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCvkpQjRnNVyk6 +NrNplQOj7L64Se3WpEJyaec0XIxC5WFfmAEdGT4Czl74T4zdNMukDgsW2Lrjg2o3 +15Sa6pPMRQB9HN3Z1+FwQoVTdOWGmawtDuc3+Bh7QoyaH2KZXtYVXbk09SyIaAQo +HkQQVCSrQiLoDsYcNQMKbBxyGTBuCvjAJRUl8CPN9KyrVa42caUJ4jO/1e/92upD +ZK1a6gNq/Zig+wypxNTJeetZZRb8vtMswdlJ7u42mG6XcFFc3Hpzsn2T1BFMO7Im +3Sx5Ezs+AUr/1pUpfGgEb6TakYvD5eirhRE1LT1hewGKoQPLkD5EFSK9/ITqcYVu +Wtproj77AgMBAAECggEAV2e+BgpWftW0gNbFTsmdgSkEbsVrtM8zrZN/vH4fS/UU +76lTMqK+NNc4Q928fRoxnjfZXU84Dl5QwbCGF8PaQy/jKMRYhGpC2jZZDdylGfUG +FJEgUPwOzN/RKGmWwPySPLO8qRqzeqOo3fK7nNKPRLhh32KmI0ndzAM6/zD0Dich +CzGO+liQbhKNoBUXlZXhJs9Bu9mvkL4yId1eks8RZAt/Opi2CoijhLhs5ncdGRR9 +oB+D8KSSyuNo3+QQBoKLz++snC6EI/NTSRM86wDroVnYVxL7mXzmQ2fWNoUmAsX4 +ugXUAmCCz1d66tAJst0l30qMykiIT/jQg4uT22ecpQKBgQDiwfBMK+t0kzcZKiNF +u7jFWNGIHnJ4EU8xdzSbaWujuE7zP0NvC0e4Ej0DiJ7S4OBhbBuSDTSl0BQUfpXZ +O/UlSNJvgvCCjGXHnvtsXzza5fqvxjhGIbO/XOhSZZhriG1YDbCtitvfP3xR85EV +Os7/2KfztR2bFH66+lj51zYn3QKBgQDb23L443avaBxD4v9DKtGIP8JS7TPm4s39 +MgluzOvupH4dWZs83RSbsBa3AkzDvU268gRjwZYLFRwld13B8sfMWEMxTVX3iliZ +58GH/4DB7ixoMLU2yJglIQ8gKrJs3nJyUh2ilU7j2HWGlRvXlMgLFrgR0udgXmnP +un9+4L7AtwKBgQCeGBDtSoGymSEFDnfhGmxMSwxrFq9APvEzz6SB5hIB53JD7Yua +VAcAstaPLCVvaRGHWtQuE77tDBGz6D2ZY1AbWpY84NHGcxRTBOjOOeg3xbft7Qk2 +/DBR7via2fJWppt1lLXliLabA6wtht9UKdhAUFFofJHTmYpx2jFGn2FGkQKBgAcc +qjNy6Dv0WjW3+hnqIK0PovGbyroFTDSMkdGLv3YQzCOW6nalkC0ZHSL+aA1M6l7J +xWqUsI3bOPXdTqMff2cU09eccNN5HjScUp5bnYaodbdJHBORmZpZ32ti9jKfX4Rj +M6I2wj6MmgCogiYPIgl5yqqI8AMmCrY1ijpIeuMxAoGAdeisS8UakUdn0lVE5NSp +stncEwnjggL5gzVAHSQ/vzSddk7mUxCWZu//GryEJeFS7Q4jhBE7z5Y6GceiBraC +6YCauAAExswgi2wpxW9dwOOY1+zKVmUIoklUJ9nIuYFSxY/pDf1rQWUOUa3x104W +FYIRNxNXs7dGOs7Ge4M01Uw= +-----END PRIVATE KEY----- diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml new file mode 100644 index 000000000000..8965b044dcbf --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml @@ -0,0 +1,99 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +spring: + banner: + charset: UTF-8 + jackson: + time-zone: UTC + date-format: "yyyy-MM-dd HH:mm:ss" + autoconfigure: + exclude: + - org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration + cloud.discovery.client.composite-indicator.enabled: false + +registry: + type: zookeeper + zookeeper: + namespace: dolphinscheduler + connect-string: localhost:2181 + retry-policy: + base-sleep-time: 1s + max-sleep: 3s + max-retries: 5 + session-timeout: 60s + connection-timeout: 15s + block-until-connected: 15s + digest: ~ + +worker: + # worker listener port + listen-port: 1234 + # worker execute thread number to limit task instances in parallel + exec-threads: 100 + # worker heartbeat interval + max-heartbeat-interval: 10s + # worker host weight to dispatch tasks, default value 100 + host-weight: 100 + server-load-protection: + # If set true, will open worker overload protection + enabled: true + # Worker max system cpu usage, when the worker's system cpu usage is smaller then this value, worker server can be dispatched tasks. + max-system-cpu-usage-percentage-thresholds: 0.7 + # Worker max jvm cpu usage, when the worker's jvm cpu usage is smaller then this value, worker server can be dispatched tasks. + max-jvm-cpu-usage-percentage-thresholds: 0.7 + # Worker max System memory usage , when the master's system memory usage is smaller then this value, master server can execute workflow. + max-system-memory-usage-percentage-thresholds: 0.7 + # Worker max disk usage , when the worker's disk usage is smaller then this value, worker server can be dispatched tasks. + max-disk-usage-percentage-thresholds: 0.7 + registry-disconnect-strategy: + # The disconnect strategy: stop, waiting + strategy: stop + task-execute-threads-full-policy: REJECT + tenant-config: + # tenant corresponds to the user of the system, which is used by the worker to submit the job. If system does not have this user, it will be automatically created after the parameter worker.tenant.auto.create is true. + auto-create-tenant-enabled: true + # If set true, will use worker bootstrap user as the tenant to execute task when the tenant is `default`. + default-tenant-enabled: false + rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem + +server: + port: 1235 + +management: + endpoints: + web: + exposure: + include: health,metrics,prometheus + endpoint: + health: + enabled: true + show-details: always + health: + db: + enabled: true + defaults: + enabled: false + metrics: + tags: + application: ${spring.application.name} + +metrics: + enabled: true From 006581aae75e465d5239af4af8489c49f483a8f5 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Wed, 16 Oct 2024 19:59:22 +0800 Subject: [PATCH 005/104] add SslShellTaskE2ETest --- .github/workflows/e2e.yml | 2 + .../e2e/cases/ssl/SslShellTaskE2ETest.java | 288 ++++++++++++++++++ 2 files changed, 290 insertions(+) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb43d4059258..03ab9a5fa2a3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,6 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java new file mode 100644 index 000000000000..c89864263d11 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.e2e.cases.tasks; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest; +import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; +import org.apache.dolphinscheduler.e2e.core.WebDriverHolder; +import org.apache.dolphinscheduler.e2e.pages.LoginPage; +import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; +import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; +import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; +import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; +import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; +import org.apache.dolphinscheduler.e2e.pages.security.UserPage; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junitpioneer.jupiter.DisableIfTestFails; + +@TestMethodOrder(MethodOrderer.MethodName.class) +@DolphinScheduler(composeFiles = "docker/gluster-test/docker-compose.yaml") +@DisableIfTestFails +public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { + + @BeforeAll + public static void setup() { + browser = WebDriverHolder.getWebDriver(); + + TenantPage tenantPage = new LoginPage(browser) + .login(adminUser) + .goToNav(SecurityPage.class) + .goToTab(TenantPage.class); + + if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) { + tenantPage + .create(adminUser.getTenant()) + .goToNav(SecurityPage.class) + .goToTab(UserPage.class) + .update(adminUser); + } + + tenantPage + .goToNav(ProjectPage.class) + .createProjectUntilSuccess(projectName); + } + + @Test + void testRunShellTasks_SuccessCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + // todo: use yaml to define the workflow + String workflowName = "SuccessCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("echo hello world\n") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_WorkflowParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + // todo: use yaml to define the workflow + String workflowName = "WorkflowParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .addGlobalParam("name", "tom") + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_LocalParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "LocalParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .addParam("name", "tom") + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "LocalParamsOverrideWorkflowParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"jerry\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .addParam("name", "tom") + .submit() + + .submit() + .name(workflowName) + .addGlobalParam("name", "jerry") + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_UsingResourceFile() { + String testFileName = "echo"; + new ResourcePage(browser) + .goToNav(ResourcePage.class) + .goToTab(FileManagePage.class) + .createFileUntilSuccess(testFileName, "echo 123"); + + final WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "UsingResourceFile"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("cat " + testFileName + ".sh") + .name(taskName) + .selectResource(testFileName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_FailedCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "FailedCase"; + String taskName = "ShellFailed"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("echo 'I am failed'\n exit1\n") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + +} From 46e9db393365cebaedbf40ae6a38a5b6bdab2223 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 11:06:55 +0800 Subject: [PATCH 006/104] add doc and fix package --- docs/docs/en/guide/installation/cluster.md | 55 +++++++++++++++++ docs/docs/zh/guide/installation/cluster.md | 59 +++++++++++++++++++ .../e2e/cases/ssl/SslShellTaskE2ETest.java | 4 +- .../api/application.yaml | 0 .../docker-compose.yml | 0 .../master/application.yaml | 0 .../{gluster-test => ssl-test}/ssl/cert.crt | 0 .../ssl/private.pem | 0 .../worker/application.yaml | 0 .../base/config/NettyServerConfig.java | 3 - 10 files changed, 116 insertions(+), 5 deletions(-) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/api/application.yaml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/docker-compose.yml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/master/application.yaml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/ssl/cert.crt (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/ssl/private.pem (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{gluster-test => ssl-test}/worker/application.yaml (100%) diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md index 5dfc65699f2b..ebd8e292a89f 100644 --- a/docs/docs/en/guide/installation/cluster.md +++ b/docs/docs/en/guide/installation/cluster.md @@ -8,6 +8,61 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that pseudo-cluster deployment is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment. +## Enable SSL (optional) +In cluster deployment, you can enable SSL authentication. Secure Sockets Layer, SSL, abbreviated as SSL, is a secure protocol that encrypts transmitted data to ensure that information is not eavesdropped or tampered with during transmission. In addition, it can authenticate servers and ensure data integrity. + +To enable SLL authentication, you have two things to do. Firstly, you need to generate `cert.crt` and `private.pem` files. + +Step 1: Install OpenSSL + +Firstly, ensure that you have installed OpenSSL. In most Linux distributions, OpenSSL is usually pre installed. If not, you can install it using the following command: + +On Ubuntu/Debian: +```bash +sudo apt-get install openssl +``` + +On CentOS/RHEL: +```bash +sudo yum install openssl +``` +Step 2: Generate private key (private.pem) + +Open the terminal and run the following command to generate a private key: + +```bash +openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 +``` + +This command will generate a 2048 bit RSA private key and save it as a private.pem file. + +Step 3: Generate Certificate Signing Request (CSR) + +Before generating a certificate, you need to generate a Certificate Signing Request (CSR). Run the following command: + +```bash +openssl req -new -key private.pem -out request.csr +``` +This command will prompt you to enter some information, such as country, state/province, organization name, etc. The information you input will be embedded into the generated certificate. + +Step 4: Generate a self signed certificate (cert.crt) + +Use CSR to generate self signed certificates. Run the following command: +```bash +openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt +``` +This command will generate a self signed certificate with a validity period of 365 days and save it as a cert.crt file. + +Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, and `dolphinscheduler-api` modules. +```yaml +rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem +``` +You need to change `enabled` to `true` and configure the file routing for `cert-file-path` and `key-file-path`. + ### Prerequisites and DolphinScheduler Startup Environment Preparations Distribute the installation package to each server of each cluster and perform all the steps in [pseudo-cluster deployment](pseudo-cluster.md) on each machine. diff --git a/docs/docs/zh/guide/installation/cluster.md b/docs/docs/zh/guide/installation/cluster.md index 5ed407ca1860..70b8e98f3d56 100644 --- a/docs/docs/zh/guide/installation/cluster.md +++ b/docs/docs/zh/guide/installation/cluster.md @@ -8,6 +8,65 @@ 集群部署(Cluster)使用的脚本和配置文件与[伪集群部署](pseudo-cluster.md)中的配置一样，所以所需要的步骤也与伪集群部署大致一样。区别就是伪集群部署针对的是一台机器，而集群部署(Cluster)需要针对多台机器，且两者“修改相关配置”步骤区别较大 +### 开启SSL（可选） +在集群部署中，你可以开启SSL认证。Secure Sockets Layer，缩写作 SSL，是一种安全协议，能够加密传输的数据，确保在数据传输过程中，信息不会被窃听或篡改，此外还可以对服务器进行身份验证以及保障数据的完整性。 + +开启SLL认证，你有两件事要做。 首先你需要生成`cert.crt`和`private.pem`文件。 + +步骤1：安装 OpenSSL + +首先，确保您已经安装了 OpenSSL。在大多数 Linux 发行版中，OpenSSL 通常已预装。如果没有，您可以通过以下命令安装它： + +在 Ubuntu/Debian 上： + +```bash +sudo apt-get install openssl +``` + +在 CentOS/RHEL 上： + +```bash +sudo yum install openssl +``` + +步骤 2：生成私钥（private.pem） + +打开终端并运行以下命令生成私钥： + +```bash +openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 +``` +此命令会生成一个 2048 位的 RSA 私钥，并将其保存为 private.pem 文件。 + +步骤 3：生成证书签署请求（CSR） + +在生成证书之前，您需要生成一个证书签署请求（CSR）。运行以下命令： + +```bash +openssl req -new -key private.pem -out request.csr +``` +此命令会提示您输入一些信息，例如国家、州/省、组织名等。您输入的信息将会嵌入到生成的证书中。 + +步骤 4：生成自签名证书（cert.crt） + +使用 CSR 来生成自签名证书。运行以下命令： + +```bash +openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt +``` +此命令会生成一个有效期为 365 天的自签名证书，并将其保存为 cert.crt 文件。 + +然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`模块中的`application.yaml`文件。 +```yaml +rpc: + ssl: + enabled: true + cert-file-path: /path/cert.crt + key-file-path: /path/private.pem +``` +您需要将`enabled`改为`true`，同时将配置`cert-file-path`和`key-file-path`的文件路劲。 + + ### 前置准备工作 && 准备 DolphinScheduler 启动环境 需要将安装包分发至每台集群的每台服务器上，并且需要在每台机器中进行配置执行[伪集群部署](pseudo-cluster.md)中的所有执行项 diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index c89864263d11..60bdc990aab3 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.e2e.cases.tasks; +package org.apache.dolphinscheduler.e2e.cases.ssl; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +42,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/gluster-test/docker-compose.yaml") +@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yaml") @DisableIfTestFails public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/api/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/docker-compose.yml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/master/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/cert.crt rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/ssl/private.pem rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/gluster-test/worker/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java index d7051dc37a47..8869b9c1debf 100644 --- a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyServerConfig.java @@ -79,9 +79,6 @@ public class NettyServerConfig { */ private int listenPort; - /** - * nettySslConfig - */ @Builder.Default private NettySslConfig nettySslConfig = new NettySslConfig(); } From 32d0edbaf2cc911c9f3747e2f1751d3d8fce00b2 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 11:13:10 +0800 Subject: [PATCH 007/104] format doc --- docs/docs/en/guide/installation/cluster.md | 9 +++++++++ docs/docs/zh/guide/installation/cluster.md | 7 ++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md index ebd8e292a89f..44eae98c2c99 100644 --- a/docs/docs/en/guide/installation/cluster.md +++ b/docs/docs/en/guide/installation/cluster.md @@ -9,6 +9,7 @@ If you are a new hand and want to experience DolphinScheduler functions, we reco Cluster deployment uses the same scripts and configuration files as [pseudo-cluster deployment](pseudo-cluster.md), so the preparation and deployment steps are the same as pseudo-cluster deployment. The difference is that pseudo-cluster deployment is for one machine, while cluster deployment (Cluster) is for multiple machines. And steps of "Modify Configuration" are quite different between pseudo-cluster deployment and cluster deployment. ## Enable SSL (optional) + In cluster deployment, you can enable SSL authentication. Secure Sockets Layer, SSL, abbreviated as SSL, is a secure protocol that encrypts transmitted data to ensure that information is not eavesdropped or tampered with during transmission. In addition, it can authenticate servers and ensure data integrity. To enable SLL authentication, you have two things to do. Firstly, you need to generate `cert.crt` and `private.pem` files. @@ -18,14 +19,17 @@ Step 1: Install OpenSSL Firstly, ensure that you have installed OpenSSL. In most Linux distributions, OpenSSL is usually pre installed. If not, you can install it using the following command: On Ubuntu/Debian: + ```bash sudo apt-get install openssl ``` On CentOS/RHEL: + ```bash sudo yum install openssl ``` + Step 2: Generate private key (private.pem) Open the terminal and run the following command to generate a private key: @@ -43,17 +47,21 @@ Before generating a certificate, you need to generate a Certificate Signing Requ ```bash openssl req -new -key private.pem -out request.csr ``` + This command will prompt you to enter some information, such as country, state/province, organization name, etc. The information you input will be embedded into the generated certificate. Step 4: Generate a self signed certificate (cert.crt) Use CSR to generate self signed certificates. Run the following command: + ```bash openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt ``` + This command will generate a self signed certificate with a validity period of 365 days and save it as a cert.crt file. Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, and `dolphinscheduler-api` modules. + ```yaml rpc: ssl: @@ -61,6 +69,7 @@ rpc: cert-file-path: /path/cert.crt key-file-path: /path/private.pem ``` + You need to change `enabled` to `true` and configure the file routing for `cert-file-path` and `key-file-path`. ### Prerequisites and DolphinScheduler Startup Environment Preparations diff --git a/docs/docs/zh/guide/installation/cluster.md b/docs/docs/zh/guide/installation/cluster.md index 70b8e98f3d56..9cdeca86ee5c 100644 --- a/docs/docs/zh/guide/installation/cluster.md +++ b/docs/docs/zh/guide/installation/cluster.md @@ -9,6 +9,7 @@ 集群部署(Cluster)使用的脚本和配置文件与[伪集群部署](pseudo-cluster.md)中的配置一样，所以所需要的步骤也与伪集群部署大致一样。区别就是伪集群部署针对的是一台机器，而集群部署(Cluster)需要针对多台机器，且两者“修改相关配置”步骤区别较大 ### 开启SSL（可选） + 在集群部署中，你可以开启SSL认证。Secure Sockets Layer，缩写作 SSL，是一种安全协议，能够加密传输的数据，确保在数据传输过程中，信息不会被窃听或篡改，此外还可以对服务器进行身份验证以及保障数据的完整性。 开启SLL认证，你有两件事要做。 首先你需要生成`cert.crt`和`private.pem`文件。 @@ -36,6 +37,7 @@ sudo yum install openssl ```bash openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 ``` + 此命令会生成一个 2048 位的 RSA 私钥，并将其保存为 private.pem 文件。 步骤 3：生成证书签署请求（CSR） @@ -45,6 +47,7 @@ openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 ```bash openssl req -new -key private.pem -out request.csr ``` + 此命令会提示您输入一些信息，例如国家、州/省、组织名等。您输入的信息将会嵌入到生成的证书中。 步骤 4：生成自签名证书（cert.crt） @@ -54,9 +57,11 @@ openssl req -new -key private.pem -out request.csr ```bash openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt ``` + 此命令会生成一个有效期为 365 天的自签名证书，并将其保存为 cert.crt 文件。 然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`模块中的`application.yaml`文件。 + ```yaml rpc: ssl: @@ -64,8 +69,8 @@ rpc: cert-file-path: /path/cert.crt key-file-path: /path/private.pem ``` -您需要将`enabled`改为`true`，同时将配置`cert-file-path`和`key-file-path`的文件路劲。 +您需要将`enabled`改为`true`，同时将配置`cert-file-path`和`key-file-path`的文件路劲。 ### 前置准备工作 && 准备 DolphinScheduler 启动环境 From 0333f83fb537f980bb834fcb744245f89deac6dd Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 12:20:27 +0800 Subject: [PATCH 008/104] change docker-compose name --- .../dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index 60bdc990aab3..2c438ba66e4a 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -42,7 +42,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yaml") +@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") @DisableIfTestFails public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { From 332fdfd52cc3431847898c1830d8c26e31c6f2f9 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 14:58:23 +0800 Subject: [PATCH 009/104] remove unnecessary content --- .../test/resources/docker/ssl-test/ssl/cert.crt | 17 ----------------- .../resources/docker/ssl-test/ssl/private.pem | 17 ----------------- 2 files changed, 34 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt index 8b26de2debbf..132745e62440 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt @@ -1,20 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -----BEGIN CERTIFICATE----- MIIEDTCCAvUCFCst8gcQmOfuqY8vJ9c4MaacVx+mMA0GCSqGSIb3DQEBCwUAMIHC MQswCQYDVQQGEwJDTjEQMA4GA1UECAwHYmVpamluZzEQMA4GA1UEBwwHYmVpamlu diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem index a28335c31a91..c0aab46c4eae 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem @@ -1,20 +1,3 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCvkpQjRnNVyk6 NrNplQOj7L64Se3WpEJyaec0XIxC5WFfmAEdGT4Czl74T4zdNMukDgsW2Lrjg2o3 From 289dd809bba909fba8d29a58570e93bf1eb99e69 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 19:04:01 +0800 Subject: [PATCH 010/104] fix e2e-test --- .../docker/ssl-test/api/application.yaml | 2 +- .../docker/ssl-test/master/application.yaml | 2 +- .../test/resources/docker/ssl-test/ssl/cert.crt | 17 +++++++++++++++++ .../resources/docker/ssl-test/ssl/private.pem | 17 +++++++++++++++++ .../docker/ssl-test/worker/application.yaml | 2 +- 5 files changed, 37 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml index 8305faa6fd3c..8ce78518c571 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml @@ -166,7 +166,7 @@ api: read-timeout: 0 rpc: ssl: - enabled: true + enabled: false cert-file-path: /path/cert.crt key-file-path: /path/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml index 979d54a02221..dc19ce8e4297 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml @@ -123,7 +123,7 @@ master: task-thread-pool-usage-weight: 40 rpc: ssl: - enabled: true + enabled: false cert-file-path: /path/cert.crt key-file-path: /path/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt index 132745e62440..12910f535382 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + -----BEGIN CERTIFICATE----- MIIEDTCCAvUCFCst8gcQmOfuqY8vJ9c4MaacVx+mMA0GCSqGSIb3DQEBCwUAMIHC MQswCQYDVQQGEwJDTjEQMA4GA1UECAwHYmVpamluZzEQMA4GA1UEBwwHYmVpamlu diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem index c0aab46c4eae..ba90f817b5c7 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem @@ -1,3 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + -----BEGIN PRIVATE KEY----- MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDCvkpQjRnNVyk6 NrNplQOj7L64Se3WpEJyaec0XIxC5WFfmAEdGT4Czl74T4zdNMukDgsW2Lrjg2o3 diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml index 8965b044dcbf..4bc55ce1ed87 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml @@ -70,7 +70,7 @@ worker: default-tenant-enabled: false rpc: ssl: - enabled: true + enabled: false cert-file-path: /path/cert.crt key-file-path: /path/private.pem From a23a59cc81acc6249f08ab3bbf814aa1beac28e6 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 19:44:33 +0800 Subject: [PATCH 011/104] change e2e-test workflow name ad task name --- .../e2e/cases/ssl/SslShellTaskE2ETest.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index 2c438ba66e4a..4745af29bd81 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -77,8 +77,8 @@ void testRunShellTasks_SuccessCase() { .goToTab(WorkflowDefinitionTab.class); // todo: use yaml to define the workflow - String workflowName = "SuccessCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslSuccessCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -112,8 +112,8 @@ void testRunShellTasks_WorkflowParamsCase() { .goToTab(WorkflowDefinitionTab.class); // todo: use yaml to define the workflow - String workflowName = "WorkflowParamsCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslWorkflowParamsCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -147,8 +147,8 @@ void testRunShellTasks_LocalParamsCase() { .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - String workflowName = "LocalParamsCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslLocalParamsCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -182,8 +182,8 @@ void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - String workflowName = "LocalParamsOverrideWorkflowParamsCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslLocalParamsOverrideWorkflowParamsCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -224,8 +224,8 @@ void testRunShellTasks_UsingResourceFile() { .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - String workflowName = "UsingResourceFile"; - String taskName = "ShellSuccess"; + String workflowName = "SslUsingResourceFile"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -259,8 +259,8 @@ void testRunShellTasks_FailedCase() { .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - String workflowName = "FailedCase"; - String taskName = "ShellFailed"; + String workflowName = "SslFailedCase"; + String taskName = "SslShellFailed"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) From be8df431a5abf5cb2b8bc6873fdce54bfc813d9b Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 19:57:07 +0800 Subject: [PATCH 012/104] delete unnecessary functions --- .../e2e/cases/ssl/SslShellTaskE2ETest.java | 184 ------------------ 1 file changed, 184 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index 4745af29bd81..b397f93ac839 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -29,8 +29,6 @@ import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; -import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; -import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; import org.apache.dolphinscheduler.e2e.pages.security.UserPage; @@ -103,186 +101,4 @@ void testRunShellTasks_SuccessCase() { assertThat(taskInstance.retryTimes()).isEqualTo(0); } - @Test - void testRunShellTasks_WorkflowParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - // todo: use yaml to define the workflow - String workflowName = "SslWorkflowParamsCase"; - String taskName = "SslShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .addGlobalParam("name", "tom") - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_LocalParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "SslLocalParamsCase"; - String taskName = "SslShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .addParam("name", "tom") - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "SslLocalParamsOverrideWorkflowParamsCase"; - String taskName = "SslShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"jerry\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .addParam("name", "tom") - .submit() - - .submit() - .name(workflowName) - .addGlobalParam("name", "jerry") - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_UsingResourceFile() { - String testFileName = "echo"; - new ResourcePage(browser) - .goToNav(ResourcePage.class) - .goToTab(FileManagePage.class) - .createFileUntilSuccess(testFileName, "echo 123"); - - final WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "SslUsingResourceFile"; - String taskName = "SslShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("cat " + testFileName + ".sh") - .name(taskName) - .selectResource(testFileName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_FailedCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "SslFailedCase"; - String taskName = "SslShellFailed"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("echo 'I am failed'\n exit1\n") - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - } From a4ed1d04c3c9afd71150a6aa497bba9f8e81f3f0 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Thu, 17 Oct 2024 20:22:07 +0800 Subject: [PATCH 013/104] change SslPythonTaskE2ETest --- .github/workflows/e2e.yml | 4 +-- ...E2ETest.java => SslPythonTaskE2ETest.java} | 33 ++++++++++++------- 2 files changed, 24 insertions(+), 13 deletions(-) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/{SslShellTaskE2ETest.java => SslPythonTaskE2ETest.java} (76%) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 03ab9a5fa2a3..a7ab7778db85 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + - name: SslPythonTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslPythonTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java similarity index 76% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java index b397f93ac839..43eea2d79e78 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java @@ -22,27 +22,29 @@ import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest; import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; import org.apache.dolphinscheduler.e2e.core.WebDriverHolder; +import org.apache.dolphinscheduler.e2e.models.environment.PythonEnvironment; import org.apache.dolphinscheduler.e2e.pages.LoginPage; import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage; import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.PythonTaskForm; +import org.apache.dolphinscheduler.e2e.pages.security.EnvironmentPage; import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; import org.apache.dolphinscheduler.e2e.pages.security.UserPage; import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Order; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; import org.junitpioneer.jupiter.DisableIfTestFails; -@TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") +@DolphinScheduler(composeFiles = "docker/ssl-task/docker-compose.yml") @DisableIfTestFails -public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { +public class SslPythonTaskE2ETest extends BaseWorkflowE2ETest { + + private static final PythonEnvironment pythonEnvironment = new PythonEnvironment(); @BeforeAll public static void setup() { @@ -60,6 +62,13 @@ public static void setup() { .goToTab(UserPage.class) .update(adminUser); } + tenantPage + .goToNav(SecurityPage.class) + .goToTab(EnvironmentPage.class) + .createEnvironmentUntilSuccess(pythonEnvironment.getEnvironmentName(), + pythonEnvironment.getEnvironmentConfig(), + pythonEnvironment.getEnvironmentDesc(), + pythonEnvironment.getEnvironmentWorkerGroup()); tenantPage .goToNav(ProjectPage.class) @@ -67,7 +76,8 @@ public static void setup() { } @Test - void testRunShellTasks_SuccessCase() { + @Order(10) + void testRunPythonTasks_SuccessCase() { WorkflowDefinitionTab workflowDefinitionPage = new ProjectPage(browser) .goToNav(ProjectPage.class) @@ -75,12 +85,13 @@ void testRunShellTasks_SuccessCase() { .goToTab(WorkflowDefinitionTab.class); // todo: use yaml to define the workflow - String workflowName = "SslSuccessCase"; - String taskName = "SslShellSuccess"; + String workflowName = "SslPythonSuccessCase"; + String taskName = "SslPythonSuccessTask"; + String pythonScripts = "print(\"success\")"; workflowDefinitionPage .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("echo hello world\n") + .addTask(WorkflowForm.TaskType.PYTHON) + .script(pythonScripts) .name(taskName) .submit() From a9638b42362ac8034cfa3531254f9afbb6cfc356 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 09:37:57 +0800 Subject: [PATCH 014/104] change SslShellTaskE2ETest --- .github/workflows/e2e.yml | 4 +- .../e2e/cases/ssl/SslPythonTaskE2ETest.java | 115 ------- .../e2e/cases/ssl/SslShellTaskE2ETest.java | 288 ++++++++++++++++++ .../docker/ssl-test/docker-compose.yml | 10 +- 4 files changed, 295 insertions(+), 122 deletions(-) delete mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a7ab7778db85..03ab9a5fa2a3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: SslPythonTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslPythonTaskE2ETest + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java deleted file mode 100644 index 43eea2d79e78..000000000000 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslPythonTaskE2ETest.java +++ /dev/null @@ -1,115 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.dolphinscheduler.e2e.cases.ssl; - -import static org.assertj.core.api.Assertions.assertThat; - -import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest; -import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; -import org.apache.dolphinscheduler.e2e.core.WebDriverHolder; -import org.apache.dolphinscheduler.e2e.models.environment.PythonEnvironment; -import org.apache.dolphinscheduler.e2e.pages.LoginPage; -import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; -import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.PythonTaskForm; -import org.apache.dolphinscheduler.e2e.pages.security.EnvironmentPage; -import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; -import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; -import org.apache.dolphinscheduler.e2e.pages.security.UserPage; - -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junitpioneer.jupiter.DisableIfTestFails; - -@DolphinScheduler(composeFiles = "docker/ssl-task/docker-compose.yml") -@DisableIfTestFails -public class SslPythonTaskE2ETest extends BaseWorkflowE2ETest { - - private static final PythonEnvironment pythonEnvironment = new PythonEnvironment(); - - @BeforeAll - public static void setup() { - browser = WebDriverHolder.getWebDriver(); - - TenantPage tenantPage = new LoginPage(browser) - .login(adminUser) - .goToNav(SecurityPage.class) - .goToTab(TenantPage.class); - - if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) { - tenantPage - .create(adminUser.getTenant()) - .goToNav(SecurityPage.class) - .goToTab(UserPage.class) - .update(adminUser); - } - tenantPage - .goToNav(SecurityPage.class) - .goToTab(EnvironmentPage.class) - .createEnvironmentUntilSuccess(pythonEnvironment.getEnvironmentName(), - pythonEnvironment.getEnvironmentConfig(), - pythonEnvironment.getEnvironmentDesc(), - pythonEnvironment.getEnvironmentWorkerGroup()); - - tenantPage - .goToNav(ProjectPage.class) - .createProjectUntilSuccess(projectName); - } - - @Test - @Order(10) - void testRunPythonTasks_SuccessCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - // todo: use yaml to define the workflow - String workflowName = "SslPythonSuccessCase"; - String taskName = "SslPythonSuccessTask"; - String pythonScripts = "print(\"success\")"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.PYTHON) - .script(pythonScripts) - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - -} diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java new file mode 100644 index 000000000000..2c438ba66e4a --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.e2e.cases.ssl; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.apache.dolphinscheduler.e2e.cases.workflow.BaseWorkflowE2ETest; +import org.apache.dolphinscheduler.e2e.core.DolphinScheduler; +import org.apache.dolphinscheduler.e2e.core.WebDriverHolder; +import org.apache.dolphinscheduler.e2e.pages.LoginPage; +import org.apache.dolphinscheduler.e2e.pages.project.ProjectPage; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.TaskInstanceTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowDefinitionTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; +import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; +import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; +import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; +import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; +import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; +import org.apache.dolphinscheduler.e2e.pages.security.UserPage; + +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.MethodOrderer; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestMethodOrder; +import org.junitpioneer.jupiter.DisableIfTestFails; + +@TestMethodOrder(MethodOrderer.MethodName.class) +@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") +@DisableIfTestFails +public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { + + @BeforeAll + public static void setup() { + browser = WebDriverHolder.getWebDriver(); + + TenantPage tenantPage = new LoginPage(browser) + .login(adminUser) + .goToNav(SecurityPage.class) + .goToTab(TenantPage.class); + + if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) { + tenantPage + .create(adminUser.getTenant()) + .goToNav(SecurityPage.class) + .goToTab(UserPage.class) + .update(adminUser); + } + + tenantPage + .goToNav(ProjectPage.class) + .createProjectUntilSuccess(projectName); + } + + @Test + void testRunShellTasks_SuccessCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + // todo: use yaml to define the workflow + String workflowName = "SuccessCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("echo hello world\n") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_WorkflowParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + // todo: use yaml to define the workflow + String workflowName = "WorkflowParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .addGlobalParam("name", "tom") + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_LocalParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "LocalParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .addParam("name", "tom") + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "LocalParamsOverrideWorkflowParamsCase"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("[ \"${name}\" = \"jerry\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") + .name(taskName) + .addParam("name", "tom") + .submit() + + .submit() + .name(workflowName) + .addGlobalParam("name", "jerry") + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_UsingResourceFile() { + String testFileName = "echo"; + new ResourcePage(browser) + .goToNav(ResourcePage.class) + .goToTab(FileManagePage.class) + .createFileUntilSuccess(testFileName, "echo 123"); + + final WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "UsingResourceFile"; + String taskName = "ShellSuccess"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("cat " + testFileName + ".sh") + .name(taskName) + .selectResource(testFileName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + + @Test + void testRunShellTasks_FailedCase() { + WorkflowDefinitionTab workflowDefinitionPage = + new ProjectPage(browser) + .goToNav(ProjectPage.class) + .goTo(projectName) + .goToTab(WorkflowDefinitionTab.class); + + String workflowName = "FailedCase"; + String taskName = "ShellFailed"; + workflowDefinitionPage + .createWorkflow() + .addTask(WorkflowForm.TaskType.SHELL) + .script("echo 'I am failed'\n exit1\n") + .name(taskName) + .submit() + + .submit() + .name(workflowName) + .submit(); + + untilWorkflowDefinitionExist(workflowName); + + workflowDefinitionPage.publish(workflowName); + + runWorkflow(workflowName); + untilWorkflowInstanceExist(workflowName); + WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName); + assertThat(workflowInstance.executionTime()).isEqualTo(1); + + TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName); + assertThat(taskInstance.retryTimes()).isEqualTo(0); + } + +} diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml index ada5261f20b1..abacd8fd35f0 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml @@ -53,7 +53,7 @@ services: - dolphinscheduler dolphinscheduler-schema-initializer: - image: ${HUB}/dolphinscheduler-tools:${TAG} + image: ${HUB}/dolphinscheduler-tools:ci env_file: .env profiles: ["schema"] command: [ tools/bin/upgrade-schema.sh ] @@ -68,7 +68,7 @@ services: - dolphinscheduler dolphinscheduler-api: - image: ${HUB}/dolphinscheduler-api:${TAG} + image: ${HUB}/dolphinscheduler-api:ci ports: - "12345:12345" - "25333:25333" @@ -93,7 +93,7 @@ services: - dolphinscheduler dolphinscheduler-alert: - image: ${HUB}/dolphinscheduler-alert-server:${TAG} + image: ${HUB}/dolphinscheduler-alert-server:ci profiles: ["all"] env_file: .env healthcheck: @@ -110,7 +110,7 @@ services: - dolphinscheduler dolphinscheduler-master: - image: ${HUB}/dolphinscheduler-master:${TAG} + image: ${HUB}/dolphinscheduler-master:ci profiles: ["all"] env_file: .env healthcheck: @@ -131,7 +131,7 @@ services: - dolphinscheduler dolphinscheduler-worker: - image: ${HUB}/dolphinscheduler-worker:${TAG} + image: ${HUB}/dolphinscheduler-worker:ci profiles: ["all"] env_file: .env healthcheck: From 60ed52072b5683dad1b15e59ce1fb134b743efef Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 09:43:58 +0800 Subject: [PATCH 015/104] change ssl-test docker-compose --- .../test/resources/docker/ssl-test/docker-compose.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml index abacd8fd35f0..1ff331ee68c2 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml @@ -53,7 +53,7 @@ services: - dolphinscheduler dolphinscheduler-schema-initializer: - image: ${HUB}/dolphinscheduler-tools:ci + image: apache/dolphinscheduler-tools:ci env_file: .env profiles: ["schema"] command: [ tools/bin/upgrade-schema.sh ] @@ -68,7 +68,7 @@ services: - dolphinscheduler dolphinscheduler-api: - image: ${HUB}/dolphinscheduler-api:ci + image: apache/dolphinscheduler-api:ci ports: - "12345:12345" - "25333:25333" @@ -93,7 +93,7 @@ services: - dolphinscheduler dolphinscheduler-alert: - image: ${HUB}/dolphinscheduler-alert-server:ci + image: apache/dolphinscheduler-alert-server:ci profiles: ["all"] env_file: .env healthcheck: @@ -110,7 +110,7 @@ services: - dolphinscheduler dolphinscheduler-master: - image: ${HUB}/dolphinscheduler-master:ci + image: apache/dolphinscheduler-master:ci profiles: ["all"] env_file: .env healthcheck: @@ -131,7 +131,7 @@ services: - dolphinscheduler dolphinscheduler-worker: - image: ${HUB}/dolphinscheduler-worker:ci + image: apache/dolphinscheduler-worker:ci profiles: ["all"] env_file: .env healthcheck: From 7a0e835edef4e36caf15d3eb48b59c2f02c9153a Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 10:51:59 +0800 Subject: [PATCH 016/104] chanage e2e-k8s.yml --- .github/workflows/e2e-k8s.yml | 6 ++++++ .github/workflows/e2e.yml | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index 30715f1dc8ba..a4475147a1d1 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -50,6 +50,12 @@ jobs: if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest timeout-minutes: 40 + strategy: + matrix: + case: + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + steps: - uses: actions/checkout@v4 with: diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 03ab9a5fa2a3..f94c3af5dbad 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,7 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: From 3bc797ead5b27a72973bc123f9ee6fd2903b60af Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 11:24:24 +0800 Subject: [PATCH 017/104] chanage --- .../e2e/cases/ssl/SslShellTaskE2ETest.java | 188 +----------------- .../docker/ssl-test/api/application.yaml | 2 +- .../docker/ssl-test/master/application.yaml | 2 +- .../docker/ssl-test/worker/application.yaml | 2 +- 4 files changed, 5 insertions(+), 189 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java index 2c438ba66e4a..b397f93ac839 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java @@ -29,8 +29,6 @@ import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowForm; import org.apache.dolphinscheduler.e2e.pages.project.workflow.WorkflowInstanceTab; import org.apache.dolphinscheduler.e2e.pages.project.workflow.task.ShellTaskForm; -import org.apache.dolphinscheduler.e2e.pages.resource.FileManagePage; -import org.apache.dolphinscheduler.e2e.pages.resource.ResourcePage; import org.apache.dolphinscheduler.e2e.pages.security.SecurityPage; import org.apache.dolphinscheduler.e2e.pages.security.TenantPage; import org.apache.dolphinscheduler.e2e.pages.security.UserPage; @@ -77,8 +75,8 @@ void testRunShellTasks_SuccessCase() { .goToTab(WorkflowDefinitionTab.class); // todo: use yaml to define the workflow - String workflowName = "SuccessCase"; - String taskName = "ShellSuccess"; + String workflowName = "SslSuccessCase"; + String taskName = "SslShellSuccess"; workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -103,186 +101,4 @@ void testRunShellTasks_SuccessCase() { assertThat(taskInstance.retryTimes()).isEqualTo(0); } - @Test - void testRunShellTasks_WorkflowParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - // todo: use yaml to define the workflow - String workflowName = "WorkflowParamsCase"; - String taskName = "ShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .addGlobalParam("name", "tom") - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_LocalParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "LocalParamsCase"; - String taskName = "ShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"tom\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .addParam("name", "tom") - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_GlobalParamsOverrideLocalParamsCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "LocalParamsOverrideWorkflowParamsCase"; - String taskName = "ShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("[ \"${name}\" = \"jerry\" ] && echo \"success\" || { echo \"failed\"; exit 1; }") - .name(taskName) - .addParam("name", "tom") - .submit() - - .submit() - .name(workflowName) - .addGlobalParam("name", "jerry") - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_UsingResourceFile() { - String testFileName = "echo"; - new ResourcePage(browser) - .goToNav(ResourcePage.class) - .goToTab(FileManagePage.class) - .createFileUntilSuccess(testFileName, "echo 123"); - - final WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "UsingResourceFile"; - String taskName = "ShellSuccess"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("cat " + testFileName + ".sh") - .name(taskName) - .selectResource(testFileName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - - @Test - void testRunShellTasks_FailedCase() { - WorkflowDefinitionTab workflowDefinitionPage = - new ProjectPage(browser) - .goToNav(ProjectPage.class) - .goTo(projectName) - .goToTab(WorkflowDefinitionTab.class); - - String workflowName = "FailedCase"; - String taskName = "ShellFailed"; - workflowDefinitionPage - .createWorkflow() - .addTask(WorkflowForm.TaskType.SHELL) - .script("echo 'I am failed'\n exit1\n") - .name(taskName) - .submit() - - .submit() - .name(workflowName) - .submit(); - - untilWorkflowDefinitionExist(workflowName); - - workflowDefinitionPage.publish(workflowName); - - runWorkflow(workflowName); - untilWorkflowInstanceExist(workflowName); - WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceFailed(workflowName); - assertThat(workflowInstance.executionTime()).isEqualTo(1); - - TaskInstanceTab.Row taskInstance = untilTaskInstanceFailed(workflowName, taskName); - assertThat(taskInstance.retryTimes()).isEqualTo(0); - } - } diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml index 8ce78518c571..8305faa6fd3c 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml @@ -166,7 +166,7 @@ api: read-timeout: 0 rpc: ssl: - enabled: false + enabled: true cert-file-path: /path/cert.crt key-file-path: /path/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml index dc19ce8e4297..979d54a02221 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml @@ -123,7 +123,7 @@ master: task-thread-pool-usage-weight: 40 rpc: ssl: - enabled: false + enabled: true cert-file-path: /path/cert.crt key-file-path: /path/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml index 4bc55ce1ed87..8965b044dcbf 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml @@ -70,7 +70,7 @@ worker: default-tenant-enabled: false rpc: ssl: - enabled: false + enabled: true cert-file-path: /path/cert.crt key-file-path: /path/private.pem From 190546b18761f33cf03475cffbdea0554dc52899 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Fri, 18 Oct 2024 16:44:58 +0800 Subject: [PATCH 018/104] fix doc and remove unnecessary modifications --- .github/workflows/e2e.yml | 1 - docs/docs/en/guide/installation/cluster.md | 27 ++++------------------ docs/docs/zh/guide/installation/cluster.md | 26 ++++----------------- 3 files changed, 10 insertions(+), 44 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f94c3af5dbad..eb43d4059258 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,7 +143,6 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/docs/docs/en/guide/installation/cluster.md b/docs/docs/en/guide/installation/cluster.md index 44eae98c2c99..301123153a71 100644 --- a/docs/docs/en/guide/installation/cluster.md +++ b/docs/docs/en/guide/installation/cluster.md @@ -10,27 +10,10 @@ Cluster deployment uses the same scripts and configuration files as [pseudo-clus ## Enable SSL (optional) -In cluster deployment, you can enable SSL authentication. Secure Sockets Layer, SSL, abbreviated as SSL, is a secure protocol that encrypts transmitted data to ensure that information is not eavesdropped or tampered with during transmission. In addition, it can authenticate servers and ensure data integrity. - +In cluster deployment, you can enable SSL for secure internal communication. The DolphinScheduler cluster can be configured to use secured communication with internal authentication of the nodes in the cluster. To enable SLL authentication, you have two things to do. Firstly, you need to generate `cert.crt` and `private.pem` files. -Step 1: Install OpenSSL - -Firstly, ensure that you have installed OpenSSL. In most Linux distributions, OpenSSL is usually pre installed. If not, you can install it using the following command: - -On Ubuntu/Debian: - -```bash -sudo apt-get install openssl -``` - -On CentOS/RHEL: - -```bash -sudo yum install openssl -``` - -Step 2: Generate private key (private.pem) +Step 1: Generate private key (private.pem) Open the terminal and run the following command to generate a private key: @@ -40,7 +23,7 @@ openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 This command will generate a 2048 bit RSA private key and save it as a private.pem file. -Step 3: Generate Certificate Signing Request (CSR) +Step 2: Generate Certificate Signing Request (CSR) Before generating a certificate, you need to generate a Certificate Signing Request (CSR). Run the following command: @@ -50,7 +33,7 @@ openssl req -new -key private.pem -out request.csr This command will prompt you to enter some information, such as country, state/province, organization name, etc. The information you input will be embedded into the generated certificate. -Step 4: Generate a self signed certificate (cert.crt) +Step 3: Generate a self signed certificate (cert.crt) Use CSR to generate self signed certificates. Run the following command: @@ -60,7 +43,7 @@ openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt This command will generate a self signed certificate with a validity period of 365 days and save it as a cert.crt file. -Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, and `dolphinscheduler-api` modules. +Then modify the `application.yaml` file in the `dolphinscheduler-master`, `dolphinscheduler-worker`, `dolphinscheduler-api` and `dolphinscheduler-alert-server` modules. ```yaml rpc: diff --git a/docs/docs/zh/guide/installation/cluster.md b/docs/docs/zh/guide/installation/cluster.md index 9cdeca86ee5c..c5aed2e3f204 100644 --- a/docs/docs/zh/guide/installation/cluster.md +++ b/docs/docs/zh/guide/installation/cluster.md @@ -10,27 +10,11 @@ ### 开启SSL（可选） -在集群部署中，你可以开启SSL认证。Secure Sockets Layer，缩写作 SSL，是一种安全协议，能够加密传输的数据，确保在数据传输过程中，信息不会被窃听或篡改，此外还可以对服务器进行身份验证以及保障数据的完整性。 +在集群部署中，您可以启用SSL以实现安全的内部通信。DolphinScheduler集群可以配置为使用安全通信，并对集群中的节点进行内部身份验证。 开启SLL认证，你有两件事要做。 首先你需要生成`cert.crt`和`private.pem`文件。 -步骤1：安装 OpenSSL - -首先，确保您已经安装了 OpenSSL。在大多数 Linux 发行版中，OpenSSL 通常已预装。如果没有，您可以通过以下命令安装它： - -在 Ubuntu/Debian 上： - -```bash -sudo apt-get install openssl -``` - -在 CentOS/RHEL 上： - -```bash -sudo yum install openssl -``` - -步骤 2：生成私钥（private.pem） +步骤 1：生成私钥（private.pem） 打开终端并运行以下命令生成私钥： @@ -40,7 +24,7 @@ openssl genpkey -algorithm RSA -out private.pem -pkeyopt rsa_keygen_bits:2048 此命令会生成一个 2048 位的 RSA 私钥，并将其保存为 private.pem 文件。 -步骤 3：生成证书签署请求（CSR） +步骤 2：生成证书签署请求（CSR） 在生成证书之前，您需要生成一个证书签署请求（CSR）。运行以下命令： @@ -50,7 +34,7 @@ openssl req -new -key private.pem -out request.csr 此命令会提示您输入一些信息，例如国家、州/省、组织名等。您输入的信息将会嵌入到生成的证书中。 -步骤 4：生成自签名证书（cert.crt） +步骤 3：生成自签名证书（cert.crt） 使用 CSR 来生成自签名证书。运行以下命令： @@ -60,7 +44,7 @@ openssl x509 -req -days 365 -in request.csr -signkey private.pem -out cert.crt 此命令会生成一个有效期为 365 天的自签名证书，并将其保存为 cert.crt 文件。 -然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`模块中的`application.yaml`文件。 +然后修改`dolphinscheduler-master`、`dolphinscheduler-worker`、`dolphinscheduler-api`、`dolphinscheduler-alert-server`模块中的`application.yaml`文件。 ```yaml rpc: From 9607b2dc19c02d260e2ff1379190936218a40b03 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Sat, 19 Oct 2024 11:50:21 +0800 Subject: [PATCH 019/104] fix --- .github/workflows/e2e-k8s.yml | 5 ---- .github/workflows/e2e.yml | 2 ++ .../extract/base/config/NettyRpcConfig.java | 29 +++++++++++++++++++ .../server/master/MasterServer.java | 7 +---- .../server/worker/WorkerServer.java | 6 +--- 5 files changed, 33 insertions(+), 16 deletions(-) create mode 100644 dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index a4475147a1d1..b8bc1b003e43 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -50,11 +50,6 @@ jobs: if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest timeout-minutes: 40 - strategy: - matrix: - case: - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest steps: - uses: actions/checkout@v4 diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb43d4059258..03ab9a5fa2a3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,6 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java new file mode 100644 index 000000000000..b83d4837dcb5 --- /dev/null +++ b/dolphinscheduler-extract/dolphinscheduler-extract-base/src/main/java/org/apache/dolphinscheduler/extract/base/config/NettyRpcConfig.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.extract.base.config; + +import lombok.Data; + +import org.springframework.context.annotation.Configuration; + +@Configuration +@Data +public class NettyRpcConfig { + + private NettySslConfig nettySslConfig; +} diff --git a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java index 953efba7c6fc..4cffd3e37224 100644 --- a/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java +++ b/dolphinscheduler-master/src/main/java/org/apache/dolphinscheduler/server/master/MasterServer.java @@ -24,7 +24,6 @@ import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; import org.apache.dolphinscheduler.dao.DaoConfiguration; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -61,8 +60,7 @@ ServiceConfiguration.class, CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class, - NettySslConfig.class}) + RegistryConfiguration.class}) @SpringBootApplication public class MasterServer implements IStoppable { @@ -96,9 +94,6 @@ public class MasterServer implements IStoppable { @Autowired private SystemEventBusFireWorker systemEventBusFireWorker; - @Autowired - NettySslConfig nettySslConfig; - public static void main(String[] args) { MasterServerMetrics.registerUncachedException(DefaultUncaughtExceptionHandler::getUncaughtExceptionCount); diff --git a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java index f833967de630..4618c5ad59b6 100644 --- a/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java +++ b/dolphinscheduler-worker/src/main/java/org/apache/dolphinscheduler/server/worker/WorkerServer.java @@ -23,7 +23,6 @@ import org.apache.dolphinscheduler.common.lifecycle.ServerLifeCycleManager; import org.apache.dolphinscheduler.common.thread.DefaultUncaughtExceptionHandler; import org.apache.dolphinscheduler.common.thread.ThreadUtils; -import org.apache.dolphinscheduler.extract.base.config.NettySslConfig; import org.apache.dolphinscheduler.meter.metrics.MetricsProvider; import org.apache.dolphinscheduler.meter.metrics.SystemMetrics; import org.apache.dolphinscheduler.plugin.datasource.api.plugin.DataSourceProcessorProvider; @@ -56,8 +55,7 @@ @Slf4j @Import({CommonConfiguration.class, StorageConfiguration.class, - RegistryConfiguration.class, - NettySslConfig.class}) + RegistryConfiguration.class}) @SpringBootApplication public class WorkerServer implements IStoppable { @@ -73,8 +71,6 @@ public class WorkerServer implements IStoppable { @Autowired private MetricsProvider metricsProvider; - @Autowired - NettySslConfig nettySslConfig; /** * worker server startup, not use web service * From 5fb575a08d3824a9a7c73f9713705814d9f46b7c Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 10:15:57 +0800 Subject: [PATCH 020/104] add ssl-e2e.yml --- .github/actions/labeler/labeler.yml | 1 + .github/workflows/e2e.yml | 2 - .github/workflows/ssl-e2e.yml | 194 ++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/ssl-e2e.yml diff --git a/.github/actions/labeler/labeler.yml b/.github/actions/labeler/labeler.yml index 6bd9b6daf1e5..c082dfbd2baa 100644 --- a/.github/actions/labeler/labeler.yml +++ b/.github/actions/labeler/labeler.yml @@ -63,6 +63,7 @@ e2e: - any: ['.github/workflows/e2e.yml'] - any: ['.github/workflows/api-test.yml'] - any: ['.github/workflows/e2e-k8s.yml'] + - any: ['.github/workflows/ssl-e2e.yml'] test: - any: ['**/test/**/*'] diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 03ab9a5fa2a3..eb43d4059258 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,8 +143,6 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml new file mode 100644 index 000000000000..33af19845133 --- /dev/null +++ b/.github/workflows/ssl-e2e.yml @@ -0,0 +1,194 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +on: + pull_request: + push: + branches: + - dev + - '[0-9]+.[0-9]+.[0-9]+-prepare' + - '[0-9]+.[0-9]+.[0-9]+-release' + +name: E2E + +concurrency: + group: e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + pull-requests: write + +jobs: + paths-filter: + name: E2E-Path-Filter + runs-on: ubuntu-latest + outputs: + not-ignore: ${{ steps.filter.outputs.not-ignore }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 + id: filter + with: + filters: | + not-ignore: + - '!(docs/**)' + build: + name: E2E-Build + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + - uses: actions/checkout@v4 + with: + submodules: true + - name: Sanity Check + uses: ./.github/actions/sanity-check + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e + restore-keys: ${{ runner.os }}-maven- + - name: Build Image + run: | + ./mvnw -B clean install \ + -Dmaven.test.skip=true \ + -Dspotless.skip=true \ + -Pdocker,staging -Ddocker.tag=ci + - name: Export Docker Images + run: | + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ + && du -sh /tmp/standalone-image.tar + done + - uses: actions/upload-artifact@v4 + name: Upload master Docker Images + with: + name: standalone-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload worker Docker Images + with: + name: standalone-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload api Docker Images + with: + name: standalone-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload tool Docker Images + with: + name: standalone-image-e2e + path: /tmp/tool-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload alert-server Docker Images + with: + name: standalone-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 + e2e: + name: ${{ matrix.case.name }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + env: + RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'adopt' + - name: Collect Workflow Telemetry + uses: ./.github/actions/workflow-telemetry-action + with: + comment_on_pr: false + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e + restore-keys: ${{ runner.os }}-maven- + - uses: actions/download-artifact@v4 + name: Download Docker Images + with: + name: standalone-image-e2e + path: /tmp + - name: Load Docker Images + run: | + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker load -i /tmp/${component}-image.tar + done + - name: Run Test + run: | + ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ + -DfailIfNoTests=false \ + -Dtest=${{ matrix.case.class }} test + - uses: actions/upload-artifact@v4 + if: always() + name: Upload Recording + with: + name: recording-${{ matrix.case.name }} + path: ${{ env.RECORDING_PATH }} + retention-days: 1 + result: + name: E2E + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [ e2e, paths-filter ] + if: always() + steps: + - name: Status + run: | + if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then + echo "Skip E2E!" + exit 0 + fi + if [[ ${{ needs.e2e.result }} != 'success' ]]; then + echo "E2E Failed!" + exit -1 + fi From 3e29c45be15ee7cbaa752619871e5c250d51aaff Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 10:50:58 +0800 Subject: [PATCH 021/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 33af19845133..a1bccdda5a36 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -23,7 +23,7 @@ on: - '[0-9]+.[0-9]+.[0-9]+-prepare' - '[0-9]+.[0-9]+.[0-9]+-release' -name: E2E +name: SSL-E2E concurrency: group: e2e-${{ github.event.pull_request.number || github.ref }} From e01ee101ac78f3e27e5bea3d7ed56bc7c0987516 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 13:59:47 +0800 Subject: [PATCH 022/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index a1bccdda5a36..371d9d801dec 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -26,7 +26,7 @@ on: name: SSL-E2E concurrency: - group: e2e-${{ github.event.pull_request.number || github.ref }} + group: ssl-e2e-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true permissions: From 6355226865aab6c4c523f6d563c05ace955c792a Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:12:55 +0800 Subject: [PATCH 023/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 371d9d801dec..e15b7b03adb0 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -34,7 +34,7 @@ permissions: jobs: paths-filter: - name: E2E-Path-Filter + name: SSL-E2E-Path-Filter runs-on: ubuntu-latest outputs: not-ignore: ${{ steps.filter.outputs.not-ignore }} @@ -47,7 +47,7 @@ jobs: not-ignore: - '!(docs/**)' build: - name: E2E-Build + name: SSL-E2E-Build needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest @@ -123,7 +123,7 @@ jobs: path: /tmp/alert-server-image.tar retention-days: 1 e2e: - name: ${{ matrix.case.name }} + name: ssl-${{ matrix.case.name }} needs: build runs-on: ubuntu-latest timeout-minutes: 30 @@ -176,7 +176,7 @@ jobs: path: ${{ env.RECORDING_PATH }} retention-days: 1 result: - name: E2E + name: ssl-E2E runs-on: ubuntu-latest timeout-minutes: 30 needs: [ e2e, paths-filter ] From 829a29d5e8ef5ba8bbecec2b0ce40de9c5919da8 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:17:50 +0800 Subject: [PATCH 024/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index e15b7b03adb0..43753af2fc47 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -122,7 +122,7 @@ jobs: name: standalone-image-e2e path: /tmp/alert-server-image.tar retention-days: 1 - e2e: + ssl-e2e: name: ssl-${{ matrix.case.name }} needs: build runs-on: ubuntu-latest @@ -179,7 +179,7 @@ jobs: name: ssl-E2E runs-on: ubuntu-latest timeout-minutes: 30 - needs: [ e2e, paths-filter ] + needs: [ ssl-e2e, paths-filter ] if: always() steps: - name: Status From 93076429d6d5cb084f37fa2ebd38fa1077021170 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:28:11 +0800 Subject: [PATCH 025/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 43753af2fc47..0db40b34fe20 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -46,7 +46,7 @@ jobs: filters: | not-ignore: - '!(docs/**)' - build: + ssl-build: name: SSL-E2E-Build needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} @@ -124,7 +124,7 @@ jobs: retention-days: 1 ssl-e2e: name: ssl-${{ matrix.case.name }} - needs: build + needs: ssl-build runs-on: ubuntu-latest timeout-minutes: 30 strategy: @@ -176,7 +176,7 @@ jobs: path: ${{ env.RECORDING_PATH }} retention-days: 1 result: - name: ssl-E2E + name: SSL-E2E runs-on: ubuntu-latest timeout-minutes: 30 needs: [ ssl-e2e, paths-filter ] @@ -185,10 +185,10 @@ jobs: - name: Status run: | if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip E2E!" + echo "Skip SSL-E2E!" exit 0 fi if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "E2E Failed!" + echo "SSL-E2E Failed!" exit -1 fi From 5a1b13da905395432708f445231f04ea4578f198 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:35:23 +0800 Subject: [PATCH 026/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 0db40b34fe20..607fdbeb5d5c 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -46,7 +46,7 @@ jobs: filters: | not-ignore: - '!(docs/**)' - ssl-build: + build: name: SSL-E2E-Build needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} @@ -124,7 +124,7 @@ jobs: retention-days: 1 ssl-e2e: name: ssl-${{ matrix.case.name }} - needs: ssl-build + needs: build runs-on: ubuntu-latest timeout-minutes: 30 strategy: From 5890d8adbc1718a24d717926ea656ed143aff54d Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 14:45:47 +0800 Subject: [PATCH 027/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 607fdbeb5d5c..a313176d6e09 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -129,6 +129,7 @@ jobs: timeout-minutes: 30 strategy: matrix: + case: - name: SslShellTaskE2ETest class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest env: From d79514ec2619daf4e8e0ddce905564b850585259 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 15:22:18 +0800 Subject: [PATCH 028/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 45 +++++-- ssl-e2e.yml | 222 ++++++++++++++++++++++++++++++++++ 2 files changed, 258 insertions(+), 9 deletions(-) create mode 100644 ssl-e2e.yml diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index a313176d6e09..078aeb630e14 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -90,36 +90,36 @@ jobs: components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ - && du -sh /tmp/standalone-image.tar + && du -sh /tmp/${component}-image.tar done - uses: actions/upload-artifact@v4 name: Upload master Docker Images with: - name: standalone-image-e2e + name: master-image-e2e path: /tmp/master-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload worker Docker Images with: - name: standalone-image-e2e + name: worker-image-e2e path: /tmp/worker-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload api Docker Images with: - name: standalone-image-e2e + name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload tool Docker Images with: - name: standalone-image-e2e + name: tool-image-e2e path: /tmp/tool-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload alert-server Docker Images with: - name: standalone-image-e2e + name: alert-server-image-e2e path: /tmp/alert-server-image.tar retention-days: 1 ssl-e2e: @@ -153,11 +153,38 @@ jobs: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e restore-keys: ${{ runner.os }}-maven- + + - uses: actions/download-artifact@v4 + name: download master Docker Images + with: + name: master-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: download api Docker Images + with: + name: api-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tool Docker Images + with: + name: tool-image-e2e + path: /tmp/tool-image.tar + retention-days: 1 - uses: actions/download-artifact@v4 - name: Download Docker Images + name: download alert-server Docker Images with: - name: standalone-image-e2e - path: /tmp + name: alert-server-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 + - name: Load Docker Images run: | components=("master" "worker" "api" "tools" "alert-server") diff --git a/ssl-e2e.yml b/ssl-e2e.yml new file mode 100644 index 000000000000..078aeb630e14 --- /dev/null +++ b/ssl-e2e.yml @@ -0,0 +1,222 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +on: + pull_request: + push: + branches: + - dev + - '[0-9]+.[0-9]+.[0-9]+-prepare' + - '[0-9]+.[0-9]+.[0-9]+-release' + +name: SSL-E2E + +concurrency: + group: ssl-e2e-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + pull-requests: write + +jobs: + paths-filter: + name: SSL-E2E-Path-Filter + runs-on: ubuntu-latest + outputs: + not-ignore: ${{ steps.filter.outputs.not-ignore }} + steps: + - uses: actions/checkout@v4 + - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 + id: filter + with: + filters: | + not-ignore: + - '!(docs/**)' + build: + name: SSL-E2E-Build + needs: paths-filter + if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + - uses: actions/checkout@v4 + with: + submodules: true + - name: Sanity Check + uses: ./.github/actions/sanity-check + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e + restore-keys: ${{ runner.os }}-maven- + - name: Build Image + run: | + ./mvnw -B clean install \ + -Dmaven.test.skip=true \ + -Dspotless.skip=true \ + -Pdocker,staging -Ddocker.tag=ci + - name: Export Docker Images + run: | + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ + && du -sh /tmp/${component}-image.tar + done + - uses: actions/upload-artifact@v4 + name: Upload master Docker Images + with: + name: master-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload api Docker Images + with: + name: api-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload tool Docker Images + with: + name: tool-image-e2e + path: /tmp/tool-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 + ssl-e2e: + name: ssl-${{ matrix.case.name }} + needs: build + runs-on: ubuntu-latest + timeout-minutes: 30 + strategy: + matrix: + case: + - name: SslShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + env: + RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} + steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Set up JDK 11 + uses: actions/setup-java@v4 + with: + java-version: 11 + distribution: 'adopt' + - name: Collect Workflow Telemetry + uses: ./.github/actions/workflow-telemetry-action + with: + comment_on_pr: false + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e + restore-keys: ${{ runner.os }}-maven- + + - uses: actions/download-artifact@v4 + name: download master Docker Images + with: + name: master-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: download api Docker Images + with: + name: api-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tool Docker Images + with: + name: tool-image-e2e + path: /tmp/tool-image.tar + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 + + - name: Load Docker Images + run: | + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker load -i /tmp/${component}-image.tar + done + - name: Run Test + run: | + ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ + -DfailIfNoTests=false \ + -Dtest=${{ matrix.case.class }} test + - uses: actions/upload-artifact@v4 + if: always() + name: Upload Recording + with: + name: recording-${{ matrix.case.name }} + path: ${{ env.RECORDING_PATH }} + retention-days: 1 + result: + name: SSL-E2E + runs-on: ubuntu-latest + timeout-minutes: 30 + needs: [ ssl-e2e, paths-filter ] + if: always() + steps: + - name: Status + run: | + if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then + echo "Skip SSL-E2E!" + exit 0 + fi + if [[ ${{ needs.e2e.result }} != 'success' ]]; then + echo "SSL-E2E Failed!" + exit -1 + fi From 614d58428060e125a7f56e4ced0da1c45be6b415 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 16:08:36 +0800 Subject: [PATCH 029/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 078aeb630e14..6bd8e5328ee5 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -81,11 +81,11 @@ jobs: restore-keys: ${{ runner.os }}-maven- - name: Build Image run: | - ./mvnw -B clean install \ + ./mvnw -B clean package \ -Dmaven.test.skip=true \ -Dspotless.skip=true \ -Pdocker,staging -Ddocker.tag=ci - - name: Export Docker Images + - name: Export Docker Imagesa run: | components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do From 452f51c8c23d04de2a1c65a47bda0fc469e3f8e2 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 17:06:35 +0800 Subject: [PATCH 030/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 6bd8e5328ee5..119bba16d2ac 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -85,11 +85,11 @@ jobs: -Dmaven.test.skip=true \ -Dspotless.skip=true \ -Pdocker,staging -Ddocker.tag=ci - - name: Export Docker Imagesa + - name: Export Docker Images run: | components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do - docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ + docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar done - uses: actions/upload-artifact@v4 From edbe7aa351237fae1bf4c41c313395342acd5695 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 19:00:03 +0800 Subject: [PATCH 031/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 119bba16d2ac..725d99766af6 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -114,7 +114,7 @@ jobs: name: Upload tool Docker Images with: name: tool-image-e2e - path: /tmp/tool-image.tar + path: /tmp/tools-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload alert-server Docker Images @@ -176,7 +176,7 @@ jobs: name: download tool Docker Images with: name: tool-image-e2e - path: /tmp/tool-image.tar + path: /tmp/tools-image.tar retention-days: 1 - uses: actions/download-artifact@v4 name: download alert-server Docker Images From e9038fd4e718c17437d4c468e15b8c5167ed2673 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Mon, 21 Oct 2024 20:46:56 +0800 Subject: [PATCH 032/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 725d99766af6..9508cdfeba07 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -81,13 +81,13 @@ jobs: restore-keys: ${{ runner.os }}-maven- - name: Build Image run: | - ./mvnw -B clean package \ + ./mvnw -B clean package -o \ -Dmaven.test.skip=true \ -Dspotless.skip=true \ -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server") + components=("master" "worker" "api" "alert-server") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar @@ -110,12 +110,6 @@ jobs: name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload tool Docker Images - with: - name: tool-image-e2e - path: /tmp/tools-image.tar - retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload alert-server Docker Images with: @@ -172,12 +166,6 @@ jobs: name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 - - uses: actions/download-artifact@v4 - name: download tool Docker Images - with: - name: tool-image-e2e - path: /tmp/tools-image.tar - retention-days: 1 - uses: actions/download-artifact@v4 name: download alert-server Docker Images with: @@ -187,7 +175,7 @@ jobs: - name: Load Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server") + components=("master" "worker" "api" "alert-server") for component in "${components[@]}"; do docker load -i /tmp/${component}-image.tar done From 6b4c53c017b6a08b370962b9bb0ff3ad596292fd Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 08:59:47 +0800 Subject: [PATCH 033/104] update ssl-e2e.yml --- .github/workflows/ssl-e2e.yml | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/ssl-e2e.yml index 9508cdfeba07..b218a6759543 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/ssl-e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 with: @@ -87,7 +87,7 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "alert-server") + components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar @@ -110,6 +110,12 @@ jobs: name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload tools Docker Images + with: + name: tool-image-e2e + path: /tmp/tools-image.tar + retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload alert-server Docker Images with: @@ -166,6 +172,12 @@ jobs: name: api-image-e2e path: /tmp/api-image.tar retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tools Docker Images + with: + name: tool-image-e2e + path: /tmp/tools-image.tar + retention-days: 1 - uses: actions/download-artifact@v4 name: download alert-server Docker Images with: @@ -175,7 +187,7 @@ jobs: - name: Load Docker Images run: | - components=("master" "worker" "api" "alert-server") + components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do docker load -i /tmp/${component}-image.tar done From 852f155ab93c8fd515463baa75b558fc77386894 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 09:52:52 +0800 Subject: [PATCH 034/104] update ssl-e2e.yml --- .github/actions/labeler/labeler.yml | 2 +- .../{ssl-e2e.yml => cluster-e2e.yml} | 34 +++++++++---------- .../ClusterShellTaskE2ETest.java} | 4 +-- 3 files changed, 20 insertions(+), 20 deletions(-) rename .github/workflows/{ssl-e2e.yml => cluster-e2e.yml} (91%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/{ssl/SslShellTaskE2ETest.java => cluster/ClusterShellTaskE2ETest.java} (97%) diff --git a/.github/actions/labeler/labeler.yml b/.github/actions/labeler/labeler.yml index c082dfbd2baa..3c951224920d 100644 --- a/.github/actions/labeler/labeler.yml +++ b/.github/actions/labeler/labeler.yml @@ -63,7 +63,7 @@ e2e: - any: ['.github/workflows/e2e.yml'] - any: ['.github/workflows/api-test.yml'] - any: ['.github/workflows/e2e-k8s.yml'] - - any: ['.github/workflows/ssl-e2e.yml'] + - any: ['.github/workflows/cluster-e2e.yml'] test: - any: ['**/test/**/*'] diff --git a/.github/workflows/ssl-e2e.yml b/.github/workflows/cluster-e2e.yml similarity index 91% rename from .github/workflows/ssl-e2e.yml rename to .github/workflows/cluster-e2e.yml index b218a6759543..dafe2a8492ee 100644 --- a/.github/workflows/ssl-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -23,10 +23,10 @@ on: - '[0-9]+.[0-9]+.[0-9]+-prepare' - '[0-9]+.[0-9]+.[0-9]+-release' -name: SSL-E2E +name: CLUSTER-E2E concurrency: - group: ssl-e2e-${{ github.event.pull_request.number || github.ref }} + group: cluster-e2e-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true permissions: @@ -34,7 +34,7 @@ permissions: jobs: paths-filter: - name: SSL-E2E-Path-Filter + name: CLUSTER-E2E-Path-Filter runs-on: ubuntu-latest outputs: not-ignore: ${{ steps.filter.outputs.not-ignore }} @@ -47,7 +47,7 @@ jobs: not-ignore: - '!(docs/**)' build: - name: SSL-E2E-Build + name: CLUSTER-E2E-Build needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest @@ -122,16 +122,16 @@ jobs: name: alert-server-image-e2e path: /tmp/alert-server-image.tar retention-days: 1 - ssl-e2e: - name: ssl-${{ matrix.case.name }} + cluster-e2e: + name: cluster-${{ matrix.case.name }} needs: build runs-on: ubuntu-latest timeout-minutes: 30 strategy: matrix: case: - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: @@ -158,31 +158,31 @@ jobs: name: download master Docker Images with: name: master-image-e2e - path: /tmp/master-image.tar + path: /tmp retention-days: 1 - uses: actions/download-artifact@v4 name: download worker Docker Images with: name: worker-image-e2e - path: /tmp/worker-image.tar + path: /tmp retention-days: 1 - uses: actions/upload-artifact@v4 name: download api Docker Images with: name: api-image-e2e - path: /tmp/api-image.tar + path: /tmp retention-days: 1 - uses: actions/download-artifact@v4 name: download tools Docker Images with: name: tool-image-e2e - path: /tmp/tools-image.tar + path: /tmp retention-days: 1 - uses: actions/download-artifact@v4 name: download alert-server Docker Images with: name: alert-server-image-e2e - path: /tmp/alert-server-image.tar + path: /tmp retention-days: 1 - name: Load Docker Images @@ -204,19 +204,19 @@ jobs: path: ${{ env.RECORDING_PATH }} retention-days: 1 result: - name: SSL-E2E + name: CLUSTER-E2E runs-on: ubuntu-latest timeout-minutes: 30 - needs: [ ssl-e2e, paths-filter ] + needs: [ cluster-e2e, paths-filter ] if: always() steps: - name: Status run: | if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip SSL-E2E!" + echo "Skip CLUSTER-E2E!" exit 0 fi if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "SSL-E2E Failed!" + echo "CLUSTER-E2E Failed!" exit -1 fi diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java similarity index 97% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index b397f93ac839..fd56561201f9 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/ssl/SslShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -15,7 +15,7 @@ * limitations under the License. */ -package org.apache.dolphinscheduler.e2e.cases.ssl; +package org.apache.dolphinscheduler.e2e.cases.cluster; import static org.assertj.core.api.Assertions.assertThat; @@ -42,7 +42,7 @@ @TestMethodOrder(MethodOrderer.MethodName.class) @DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") @DisableIfTestFails -public class SslShellTaskE2ETest extends BaseWorkflowE2ETest { +public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { @BeforeAll public static void setup() { From 80cc92d8603ee46dd1229274f10558fc35fc685d Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 10:44:37 +0800 Subject: [PATCH 035/104] increase build time --- .github/workflows/cluster-e2e.yml | 4 +--- .github/workflows/e2e.yml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index dafe2a8492ee..7cc736b9dabe 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -153,7 +153,6 @@ jobs: path: ~/.m2/repository key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e restore-keys: ${{ runner.os }}-maven- - - uses: actions/download-artifact@v4 name: download master Docker Images with: @@ -175,7 +174,7 @@ jobs: - uses: actions/download-artifact@v4 name: download tools Docker Images with: - name: tool-image-e2e + name: tools-image-e2e path: /tmp retention-days: 1 - uses: actions/download-artifact@v4 @@ -184,7 +183,6 @@ jobs: name: alert-server-image-e2e path: /tmp retention-days: 1 - - name: Load Docker Images run: | components=("master" "worker" "api" "tools" "alert-server") diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb43d4059258..b63a0fa288d0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 with: From c24b79c5d333000db0fc17c103123f25a1b5afd4 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 11:13:40 +0800 Subject: [PATCH 036/104] update --- .github/workflows/cluster-e2e.yml | 34 +++++-------------------------- .github/workflows/e2e.yml | 2 +- 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 7cc736b9dabe..a488a02f0964 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -130,8 +130,8 @@ jobs: strategy: matrix: case: - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: @@ -154,33 +154,9 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e restore-keys: ${{ runner.os }}-maven- - uses: actions/download-artifact@v4 - name: download master Docker Images + name: download cluster Docker Images with: - name: master-image-e2e - path: /tmp - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download worker Docker Images - with: - name: worker-image-e2e - path: /tmp - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: download api Docker Images - with: - name: api-image-e2e - path: /tmp - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download tools Docker Images - with: - name: tools-image-e2e - path: /tmp - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download alert-server Docker Images - with: - name: alert-server-image-e2e + name: cluster-image-e2e path: /tmp retention-days: 1 - name: Load Docker Images @@ -217,4 +193,4 @@ jobs: if [[ ${{ needs.e2e.result }} != 'success' ]]; then echo "CLUSTER-E2E Failed!" exit -1 - fi + fi \ No newline at end of file diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b63a0fa288d0..eb43d4059258 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 20 steps: - uses: actions/checkout@v4 with: From 37797c6fec5e0b70b4d9c56fc14e7c02dbc50691 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 14:15:37 +0800 Subject: [PATCH 037/104] update --- .github/workflows/cluster-e2e.yml | 36 +++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index a488a02f0964..20ff80099949 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -130,8 +130,8 @@ jobs: strategy: matrix: case: - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: @@ -154,16 +154,40 @@ jobs: key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e restore-keys: ${{ runner.os }}-maven- - uses: actions/download-artifact@v4 - name: download cluster Docker Images + name: download master Docker Images with: - name: cluster-image-e2e - path: /tmp + name: master-image-e2e + path: /tmp/master + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: download api Docker Images + with: + name: api-image-e2e + path: /tmp/api + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tools Docker Images + with: + name: tools-image-e2e + path: /tmp/tools + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server retention-days: 1 - name: Load Docker Images run: | components=("master" "worker" "api" "tools" "alert-server") for component in "${components[@]}"; do - docker load -i /tmp/${component}-image.tar + docker load -i /tmp/${component}/${component}-image.tar done - name: Run Test run: | From c85215b91c9104ecf288491480705c5cd80ab283 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 14:51:32 +0800 Subject: [PATCH 038/104] update --- .github/workflows/cluster-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 20ff80099949..0158b77cf25b 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -113,7 +113,7 @@ jobs: - uses: actions/upload-artifact@v4 name: Upload tools Docker Images with: - name: tool-image-e2e + name: tools-image-e2e path: /tmp/tools-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 From d53d0502ad6d05fb326f2371fa3251fa732a01fe Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 15:40:55 +0800 Subject: [PATCH 039/104] update --- .github/workflows/cluster-e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 0158b77cf25b..987c93827041 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -171,6 +171,7 @@ jobs: name: api-image-e2e path: /tmp/api retention-days: 1 + overwrite: true - uses: actions/download-artifact@v4 name: download tools Docker Images with: From 734038ce5bc609f40d9813baa79e42d5e504af02 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 15:42:01 +0800 Subject: [PATCH 040/104] update --- .github/workflows/cluster-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 987c93827041..2c9d66429d28 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -165,7 +165,7 @@ jobs: name: worker-image-e2e path: /tmp/worker retention-days: 1 - - uses: actions/upload-artifact@v4 + - uses: actions/download-artifact@v4 name: download api Docker Images with: name: api-image-e2e From 5d836e90db35d469788d83906eef16333d4d0b9e Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 16:19:14 +0800 Subject: [PATCH 041/104] update --- .github/workflows/cluster-e2e.yml | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 2c9d66429d28..65923db3a368 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -138,6 +138,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' - name: Set up JDK 11 uses: actions/setup-java@v4 with: @@ -171,7 +181,6 @@ jobs: name: api-image-e2e path: /tmp/api retention-days: 1 - overwrite: true - uses: actions/download-artifact@v4 name: download tools Docker Images with: From e403cb014c8d6e20f7763ada50d75ee6e12f948a Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 23:25:45 +0800 Subject: [PATCH 042/104] update --- .github/workflows/cluster-e2e.yml | 20 +- .../cluster/ClusterShellTaskE2ETest.java | 2 +- .../api/application.yaml | 0 .../docker-compose.yml | 0 .../master/application.yaml | 0 .../{ssl-test => cluster-test}/ssl/cert.crt | 0 .../ssl/private.pem | 0 .../worker/application.yaml | 0 ssl-e2e.yml | 222 ------------------ 9 files changed, 6 insertions(+), 238 deletions(-) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/api/application.yaml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/docker-compose.yml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/master/application.yaml (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/ssl/cert.crt (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/ssl/private.pem (100%) rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/{ssl-test => cluster-test}/worker/application.yaml (100%) delete mode 100644 ssl-e2e.yml diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 65923db3a368..703797fe0e78 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 30 + timeout-minutes: 40 steps: - uses: actions/checkout@v4 with: @@ -59,8 +59,8 @@ jobs: - name: Maximize runner space uses: ./.github/actions/maximize-build-space with: - root-reserve-mb: 30720 - temp-reserve-mb: 10240 + root-reserve-mb: 40720 + temp-reserve-mb: 20240 remove-dotnet: 'true' remove-android: 'true' remove-haskell: 'true' @@ -130,24 +130,14 @@ jobs: strategy: matrix: case: - - name: ClusterShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest + - name: ClusterShellTaskE2ETest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: - uses: actions/checkout@v4 with: submodules: true - - name: Maximize runner space - uses: ./.github/actions/maximize-build-space - with: - root-reserve-mb: 30720 - temp-reserve-mb: 10240 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' - name: Set up JDK 11 uses: actions/setup-java@v4 with: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index fd56561201f9..9e93b3ba3bfb 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -40,7 +40,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/ssl-test/docker-compose.yml") +@DolphinScheduler(composeFiles = "docker/cluster-test/docker-compose.yml") @DisableIfTestFails public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/api/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/api/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/docker-compose.yml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/master/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/master/application.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/cert.crt similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/cert.crt rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/cert.crt diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/private.pem similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/ssl/private.pem rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/ssl/private.pem diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/application.yaml similarity index 100% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/ssl-test/worker/application.yaml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/worker/application.yaml diff --git a/ssl-e2e.yml b/ssl-e2e.yml deleted file mode 100644 index 078aeb630e14..000000000000 --- a/ssl-e2e.yml +++ /dev/null @@ -1,222 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -on: - pull_request: - push: - branches: - - dev - - '[0-9]+.[0-9]+.[0-9]+-prepare' - - '[0-9]+.[0-9]+.[0-9]+-release' - -name: SSL-E2E - -concurrency: - group: ssl-e2e-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true - -permissions: - pull-requests: write - -jobs: - paths-filter: - name: SSL-E2E-Path-Filter - runs-on: ubuntu-latest - outputs: - not-ignore: ${{ steps.filter.outputs.not-ignore }} - steps: - - uses: actions/checkout@v4 - - uses: dorny/paths-filter@b2feaf19c27470162a626bd6fa8438ae5b263721 - id: filter - with: - filters: | - not-ignore: - - '!(docs/**)' - build: - name: SSL-E2E-Build - needs: paths-filter - if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} - runs-on: ubuntu-latest - timeout-minutes: 20 - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Maximize runner space - uses: ./.github/actions/maximize-build-space - with: - root-reserve-mb: 30720 - temp-reserve-mb: 10240 - remove-dotnet: 'true' - remove-android: 'true' - remove-haskell: 'true' - remove-codeql: 'true' - remove-docker-images: 'true' - - uses: actions/checkout@v4 - with: - submodules: true - - name: Sanity Check - uses: ./.github/actions/sanity-check - with: - token: ${{ secrets.GITHUB_TOKEN }} - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - name: Build Image - run: | - ./mvnw -B clean install \ - -Dmaven.test.skip=true \ - -Dspotless.skip=true \ - -Pdocker,staging -Ddocker.tag=ci - - name: Export Docker Images - run: | - components=("master" "worker" "api" "tools" "alert-server") - for component in "${components[@]}"; do - docker save apache/dolphinscheduler-${component}-server:ci -o /tmp/${component}-image.tar \ - && du -sh /tmp/${component}-image.tar - done - - uses: actions/upload-artifact@v4 - name: Upload master Docker Images - with: - name: master-image-e2e - path: /tmp/master-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload worker Docker Images - with: - name: worker-image-e2e - path: /tmp/worker-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload api Docker Images - with: - name: api-image-e2e - path: /tmp/api-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload tool Docker Images - with: - name: tool-image-e2e - path: /tmp/tool-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload alert-server Docker Images - with: - name: alert-server-image-e2e - path: /tmp/alert-server-image.tar - retention-days: 1 - ssl-e2e: - name: ssl-${{ matrix.case.name }} - needs: build - runs-on: ubuntu-latest - timeout-minutes: 30 - strategy: - matrix: - case: - - name: SslShellTaskE2ETest - class: org.apache.dolphinscheduler.e2e.cases.ssl.SslShellTaskE2ETest - env: - RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} - steps: - - uses: actions/checkout@v4 - with: - submodules: true - - name: Set up JDK 11 - uses: actions/setup-java@v4 - with: - java-version: 11 - distribution: 'adopt' - - name: Collect Workflow Telemetry - uses: ./.github/actions/workflow-telemetry-action - with: - comment_on_pr: false - - name: Cache local Maven repository - uses: actions/cache@v4 - with: - path: ~/.m2/repository - key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e - restore-keys: ${{ runner.os }}-maven- - - - uses: actions/download-artifact@v4 - name: download master Docker Images - with: - name: master-image-e2e - path: /tmp/master-image.tar - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download worker Docker Images - with: - name: worker-image-e2e - path: /tmp/worker-image.tar - retention-days: 1 - - uses: actions/upload-artifact@v4 - name: download api Docker Images - with: - name: api-image-e2e - path: /tmp/api-image.tar - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download tool Docker Images - with: - name: tool-image-e2e - path: /tmp/tool-image.tar - retention-days: 1 - - uses: actions/download-artifact@v4 - name: download alert-server Docker Images - with: - name: alert-server-image-e2e - path: /tmp/alert-server-image.tar - retention-days: 1 - - - name: Load Docker Images - run: | - components=("master" "worker" "api" "tools" "alert-server") - for component in "${components[@]}"; do - docker load -i /tmp/${component}-image.tar - done - - name: Run Test - run: | - ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ - -DfailIfNoTests=false \ - -Dtest=${{ matrix.case.class }} test - - uses: actions/upload-artifact@v4 - if: always() - name: Upload Recording - with: - name: recording-${{ matrix.case.name }} - path: ${{ env.RECORDING_PATH }} - retention-days: 1 - result: - name: SSL-E2E - runs-on: ubuntu-latest - timeout-minutes: 30 - needs: [ ssl-e2e, paths-filter ] - if: always() - steps: - - name: Status - run: | - if [[ ${{ needs.paths-filter.outputs.not-ignore }} == 'false' && ${{ github.event_name }} == 'pull_request' ]]; then - echo "Skip SSL-E2E!" - exit 0 - fi - if [[ ${{ needs.e2e.result }} != 'success' ]]; then - echo "SSL-E2E Failed!" - exit -1 - fi From d642c1f5acc296c61c13849a0ee337e0c2b3bee6 Mon Sep 17 00:00:00 2001 From: xdu-chenrj Date: Tue, 22 Oct 2024 23:30:32 +0800 Subject: [PATCH 043/104] update --- .github/workflows/cluster-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 703797fe0e78..bbdd7ddafd78 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -60,7 +60,7 @@ jobs: uses: ./.github/actions/maximize-build-space with: root-reserve-mb: 40720 - temp-reserve-mb: 20240 + temp-reserve-mb: 10240 remove-dotnet: 'true' remove-android: 'true' remove-haskell: 'true' From 3c9eb2437f2089d9928650028cd58a256ab3d0d6 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 09:23:21 +0800 Subject: [PATCH 044/104] add cluster-e2e --- .github/workflows/cluster-e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index bbdd7ddafd78..e9bc50576699 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -59,7 +59,7 @@ jobs: - name: Maximize runner space uses: ./.github/actions/maximize-build-space with: - root-reserve-mb: 40720 + root-reserve-mb: 30720 temp-reserve-mb: 10240 remove-dotnet: 'true' remove-android: 'true' @@ -217,4 +217,4 @@ jobs: if [[ ${{ needs.e2e.result }} != 'success' ]]; then echo "CLUSTER-E2E Failed!" exit -1 - fi \ No newline at end of file + fi From 2b94eede14ae975ec744ca9a6cb38b1ba28e0004 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 09:45:02 +0800 Subject: [PATCH 045/104] modify mvn command --- .github/workflows/cluster-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index e9bc50576699..5d49008e2576 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -81,7 +81,7 @@ jobs: restore-keys: ${{ runner.os }}-maven- - name: Build Image run: | - ./mvnw -B clean package -o \ + ./mvnw -B clean package \ -Dmaven.test.skip=true \ -Dspotless.skip=true \ -Pdocker,staging -Ddocker.tag=ci From fc8826cdf545c5d6c8b376b139ee9c73b632d901 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 10:15:24 +0800 Subject: [PATCH 046/104] fix no space left on device --- .github/workflows/cluster-e2e.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 5d49008e2576..92840baf80ef 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -138,6 +138,16 @@ jobs: - uses: actions/checkout@v4 with: submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' - name: Set up JDK 11 uses: actions/setup-java@v4 with: From 43a3132ae6f738d5430770219d1c89e6b3275b21 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 10:38:43 +0800 Subject: [PATCH 047/104] add actions/checkout@v4 --- .github/workflows/cluster-e2e.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/cluster-e2e.yml b/.github/workflows/cluster-e2e.yml index 92840baf80ef..df5ffb44f8c9 100644 --- a/.github/workflows/cluster-e2e.yml +++ b/.github/workflows/cluster-e2e.yml @@ -148,6 +148,9 @@ jobs: remove-haskell: 'true' remove-codeql: 'true' remove-docker-images: 'true' + - uses: actions/checkout@v4 + with: + submodules: true - name: Set up JDK 11 uses: actions/setup-java@v4 with: From daa809dee1b28c61391e87abf63472f3574f5fc5 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 12:18:00 +0800 Subject: [PATCH 048/104] ut --- .../cluster/ClusterShellTaskE2ETest.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index 9e93b3ba3bfb..ce41db388f78 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -46,37 +46,46 @@ public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { @BeforeAll public static void setup() { + System.out.println("-----start setup------"); browser = WebDriverHolder.getWebDriver(); + System.out.println("-----start setup1------"); TenantPage tenantPage = new LoginPage(browser) .login(adminUser) .goToNav(SecurityPage.class) .goToTab(TenantPage.class); + System.out.println("-----start setup2------"); if (tenantPage.tenants().stream().noneMatch(tenant -> tenant.tenantCode().equals(adminUser.getTenant()))) { + System.out.println("-----start setup3------"); tenantPage .create(adminUser.getTenant()) .goToNav(SecurityPage.class) .goToTab(UserPage.class) .update(adminUser); } + System.out.println("-----start setup4------"); tenantPage .goToNav(ProjectPage.class) .createProjectUntilSuccess(projectName); + System.out.println("-----start setup5------"); } @Test void testRunShellTasks_SuccessCase() { + System.out.println("-----start------"); WorkflowDefinitionTab workflowDefinitionPage = new ProjectPage(browser) .goToNav(ProjectPage.class) .goTo(projectName) .goToTab(WorkflowDefinitionTab.class); - + System.out.println("-----start1------"); // todo: use yaml to define the workflow String workflowName = "SslSuccessCase"; + System.out.println("-----start2------"); String taskName = "SslShellSuccess"; + System.out.println("-----start3------"); workflowDefinitionPage .createWorkflow() .addTask(WorkflowForm.TaskType.SHELL) @@ -87,17 +96,25 @@ void testRunShellTasks_SuccessCase() { .submit() .name(workflowName) .submit(); + System.out.println("-----start4------"); untilWorkflowDefinitionExist(workflowName); + System.out.println("-----start5------"); workflowDefinitionPage.publish(workflowName); + System.out.println("-----start6------"); runWorkflow(workflowName); + System.out.println("-----start7------"); untilWorkflowInstanceExist(workflowName); + System.out.println("-----start8------"); WorkflowInstanceTab.Row workflowInstance = untilWorkflowInstanceSuccess(workflowName); + System.out.println("-----start9------"); assertThat(workflowInstance.executionTime()).isEqualTo(1); + System.out.println("-----start10------"); TaskInstanceTab.Row taskInstance = untilTaskInstanceSuccess(workflowName, taskName); + System.out.println("-----start11------"); assertThat(taskInstance.retryTimes()).isEqualTo(0); } From ebc5c667d6b3ee95d82697b1b67a3428f426a1e4 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 13:30:27 +0800 Subject: [PATCH 049/104] UT --- .../e2e/cases/cluster/ClusterShellTaskE2ETest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index ce41db388f78..25edcb203885 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -40,7 +40,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/cluster-test/docker-compose.yml") +@DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml") @DisableIfTestFails public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { From ec57c9395f6db6718c8135b21c2e38daf05bdd9c Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 14:53:28 +0800 Subject: [PATCH 050/104] test --- .github/workflows/e2e.yml | 2 + .../cluster/ClusterShellTaskE2ETest.java | 3 +- .../docker/basic/docker-compose.yaml | 2 +- .../cluster-test/docker-compose-api.yaml | 37 ++++++++++++++++++ ...ker-compose.yml => docker-compose-bak.yml} | 13 ++++--- .../cluster-test/docker-compose-master.yaml | 38 +++++++++++++++++++ .../cluster-test/docker-compose-worker.yaml | 38 +++++++++++++++++++ 7 files changed, 125 insertions(+), 8 deletions(-) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-api.yaml rename dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/{docker-compose.yml => docker-compose-bak.yml} (97%) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-master.yaml create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-worker.yaml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index eb43d4059258..1f73d46ff153 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -143,6 +143,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.SqlServerDataSourceE2ETest - name: HiveDataSourceE2ETest class: org.apache.dolphinscheduler.e2e.cases.HiveDataSourceE2ETest + - name: SSLTest + class: org.apache.dolphinscheduler.e2e.cases.cluster.ClusterShellTaskE2ETest env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index 25edcb203885..ffaaa6a445fc 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -40,7 +40,8 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = "docker/basic/docker-compose.yaml") +@DolphinScheduler(composeFiles = {"docker/cluster-test/docker-compose-api.yaml", + "docker/cluster-test/docker-compose-master.yaml", "docker/cluster-test/docker-compose-worker.yaml"}) @DisableIfTestFails public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml index 37bebedc647b..85654dbabf77 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml @@ -32,6 +32,6 @@ services: interval: 5s timeout: 5s retries: 120 - + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] networks: e2e: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-api.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-api.yaml new file mode 100644 index 000000000000..85654dbabf77 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-api.yaml @@ -0,0 +1,37 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] +networks: + e2e: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-bak.yml similarity index 97% rename from dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml rename to dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-bak.yml index 1ff331ee68c2..ac048b5d1d7e 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-bak.yml @@ -34,7 +34,7 @@ services: timeout: 60s retries: 120 networks: - - dolphinscheduler + - e2e dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 @@ -50,7 +50,7 @@ services: timeout: 60s retries: 120 networks: - - dolphinscheduler + - e2e dolphinscheduler-schema-initializer: image: apache/dolphinscheduler-tools:ci @@ -65,7 +65,7 @@ services: - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler networks: - - dolphinscheduler + - e2e dolphinscheduler-api: image: apache/dolphinscheduler-api:ci @@ -90,7 +90,8 @@ services: - ./ssl/cert.crt:/path/cert.crt - ./ssl/private.pm:/path/private.pm networks: - - dolphinscheduler + - e2e + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: image: apache/dolphinscheduler-alert-server:ci @@ -128,7 +129,7 @@ services: - ./ssl/cert.crt:/path/cert.crt - ./ssl/private.pm:/path/private.pm networks: - - dolphinscheduler + - e2e dolphinscheduler-worker: image: apache/dolphinscheduler-worker:ci @@ -151,7 +152,7 @@ services: - ./ssl/cert.crt:/path/cert.crt - ./ssl/private.pm:/path/private.pm networks: - - dolphinscheduler + - e2e networks: dolphinscheduler: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-master.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-master.yaml new file mode 100644 index 000000000000..758a2cbf88bd --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-master.yaml @@ -0,0 +1,38 @@ + docker-compose-api.yaml# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "5678:5678" + - "5679:5679" + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] +networks: + e2e: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-worker.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-worker.yaml new file mode 100644 index 000000000000..806787bba7a7 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-worker.yaml @@ -0,0 +1,38 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "1234:1234" + - "1235:1235" + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +networks: + e2e: From ceede2ea12abfd05e9ea81450b099c8c974ede78 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 15:14:55 +0800 Subject: [PATCH 051/104] fix code error --- .../src/test/resources/docker/basic/docker-compose.yaml | 2 +- .../docker/cluster-test/docker-compose-master.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml index 85654dbabf77..37bebedc647b 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml @@ -32,6 +32,6 @@ services: interval: 5s timeout: 5s retries: 120 - command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + networks: e2e: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-master.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-master.yaml index 758a2cbf88bd..aa276adfafbd 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-master.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-master.yaml @@ -1,4 +1,4 @@ - docker-compose-api.yaml# +# # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -24,15 +24,15 @@ services: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' ports: - - "5678:5678" - - "5679:5679" + - "12345:12345" networks: - e2e healthcheck: - test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] networks: e2e: + From 304f560e72238a0f067e6f91e6440d3b31e5a44b Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 16:10:55 +0800 Subject: [PATCH 052/104] test --- .../cluster/ClusterShellTaskE2ETest.java | 3 +- .../cluster-test/docker-compose-zk.yaml | 35 +++++++++++++++++++ 2 files changed, 37 insertions(+), 1 deletion(-) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-zk.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index ffaaa6a445fc..ae39770bc550 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -41,7 +41,8 @@ @TestMethodOrder(MethodOrderer.MethodName.class) @DolphinScheduler(composeFiles = {"docker/cluster-test/docker-compose-api.yaml", - "docker/cluster-test/docker-compose-master.yaml", "docker/cluster-test/docker-compose-worker.yaml"}) + "docker/cluster-test/docker-compose-master.yaml", "docker/cluster-test/docker-compose-worker.yaml", + "docker/cluster-test/docker-compose-zk.yaml"}) @DisableIfTestFails public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-zk.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-zk.yaml new file mode 100644 index 000000000000..205a5203b9b1 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose-zk.yaml @@ -0,0 +1,35 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler-zookeeper: + image: bitnami/zookeeper:3.7.1 + profiles: [ "all" ] + environment: + ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181" ] + interval: 5s + timeout: 60s + retries: 120 +networks: + e2e: From 951460a6e319270246fb506bd3464c14feb06525 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 16:39:27 +0800 Subject: [PATCH 053/104] test --- .../cluster/ClusterShellTaskE2ETest.java | 4 +- .../docker/cluster-test/docker-compose.yaml | 82 +++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java index ae39770bc550..abf3f79e9c7e 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/java/org/apache/dolphinscheduler/e2e/cases/cluster/ClusterShellTaskE2ETest.java @@ -40,9 +40,7 @@ import org.junitpioneer.jupiter.DisableIfTestFails; @TestMethodOrder(MethodOrderer.MethodName.class) -@DolphinScheduler(composeFiles = {"docker/cluster-test/docker-compose-api.yaml", - "docker/cluster-test/docker-compose-master.yaml", "docker/cluster-test/docker-compose-worker.yaml", - "docker/cluster-test/docker-compose-zk.yaml"}) +@DolphinScheduler(composeFiles = "docker/cluster-test/docker-compose.yaml") @DisableIfTestFails public class ClusterShellTaskE2ETest extends BaseWorkflowE2ETest { diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml new file mode 100644 index 000000000000..e56b895f008b --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -0,0 +1,82 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +version: "3.8" + +services: + dolphinscheduler-zookeeper: + image: bitnami/zookeeper:3.7.1 + profiles: [ "all" ] + environment: + ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181" ] + interval: 5s + timeout: 60s + retries: 120 + dolphinscheduler-api: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + depends_on: + - dolphinscheduler-zookeeper + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + dolphinscheduler-master: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "5678:5678" + - "5679:5679" + depends_on: + - dolphinscheduler-zookeeper + healthcheck: + test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] + dolphinscheduler-worker: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "1234:1234" + - "1235:1235" + depends_on: + - dolphinscheduler-zookeeper + healthcheck: + test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +networks: + e2e: From d3dc75705839f691f1439da368345e8f69757927 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 17:08:30 +0800 Subject: [PATCH 054/104] test --- .../resources/docker/cluster-test/docker-compose.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index e56b895f008b..a1218ef94fa1 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -39,7 +39,8 @@ services: ports: - "12345:12345" depends_on: - - dolphinscheduler-zookeeper + dolphinscheduler-zookeeper: + condition: service_healthy healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s @@ -55,7 +56,8 @@ services: - "5678:5678" - "5679:5679" depends_on: - - dolphinscheduler-zookeeper + dolphinscheduler-zookeeper: + condition: service_healthy healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 5s @@ -71,7 +73,8 @@ services: - "1234:1234" - "1235:1235" depends_on: - - dolphinscheduler-zookeeper + dolphinscheduler-zookeeper: + condition: service_healthy healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 5s From ac558d0ddabafca42d89326f6bfa5bb709ff80f0 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 17:36:38 +0800 Subject: [PATCH 055/104] test docker-compose.yaml --- .../test/resources/docker/cluster-test/docker-compose.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index a1218ef94fa1..72183e318e03 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,6 +31,11 @@ services: interval: 5s timeout: 60s retries: 120 + networks: + - e2e + deploy: + mode: replicated + replicas: 1 dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci environment: From e30afdc6f622f0b781a082602e72877e235b632d Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 17:51:08 +0800 Subject: [PATCH 056/104] test docker-compose1 --- .../docker/cluster-test/docker-compose.yaml | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 72183e318e03..f5fc675c87dc 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -36,6 +36,7 @@ services: deploy: mode: replicated replicas: 1 + dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci environment: @@ -46,12 +47,15 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy + networks: + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + dolphinscheduler-master: image: apache/dolphinscheduler-standalone-server:ci environment: @@ -63,12 +67,15 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy + networks: + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 5s timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] + dolphinscheduler-worker: image: apache/dolphinscheduler-standalone-server:ci environment: @@ -80,11 +87,12 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy + networks: + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 5s timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] -networks: - e2e: + From 1da1118cd9bcf15361d1ada5b48c697b5b5d0962 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Wed, 23 Oct 2024 18:08:42 +0800 Subject: [PATCH 057/104] modify network --- .../src/test/resources/docker/cluster-test/docker-compose.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index f5fc675c87dc..b12bf140c0f2 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -96,3 +96,5 @@ services: retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +networks: + e2e: From 0a0e244c3a977ca5d084e27cdd04245f14396d70 Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Wed, 23 Oct 2024 21:54:51 +0800 Subject: [PATCH 058/104] Update docker-compose.yaml --- .../docker/cluster-test/docker-compose.yaml | 63 ------------------- 1 file changed, 63 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index b12bf140c0f2..51d731cf0a0c 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -33,68 +33,5 @@ services: retries: 120 networks: - e2e - deploy: - mode: replicated - replicas: 1 - - dolphinscheduler-api: - image: apache/dolphinscheduler-standalone-server:ci - environment: - MASTER_MAX_CPU_LOAD_AVG: 100 - WORKER_TENANT_AUTO_CREATE: 'true' - ports: - - "12345:12345" - depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy - networks: - - e2e - healthcheck: - test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] - interval: 5s - timeout: 5s - retries: 120 - command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] - - dolphinscheduler-master: - image: apache/dolphinscheduler-standalone-server:ci - environment: - MASTER_MAX_CPU_LOAD_AVG: 100 - WORKER_TENANT_AUTO_CREATE: 'true' - ports: - - "5678:5678" - - "5679:5679" - depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy - networks: - - e2e - healthcheck: - test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] - interval: 5s - timeout: 5s - retries: 120 - command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] - - dolphinscheduler-worker: - image: apache/dolphinscheduler-standalone-server:ci - environment: - MASTER_MAX_CPU_LOAD_AVG: 100 - WORKER_TENANT_AUTO_CREATE: 'true' - ports: - - "1234:1234" - - "1235:1235" - depends_on: - dolphinscheduler-zookeeper: - condition: service_healthy - networks: - - e2e - healthcheck: - test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] - interval: 5s - timeout: 5s - retries: 120 - command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] - networks: e2e: From 0d9a555857b91565ed93c07821cc6b3fb3309b81 Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Wed, 23 Oct 2024 22:28:30 +0800 Subject: [PATCH 059/104] Update docker-compose.yaml --- .../docker/cluster-test/docker-compose.yaml | 59 +++++++++++++++++++ 1 file changed, 59 insertions(+) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 51d731cf0a0c..7f60778997df 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,7 +31,66 @@ services: interval: 5s timeout: 60s retries: 120 + command: ["sh","-c","/opt/apache/bootstrap.sh"] networks: - e2e + dolphinscheduler-api: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "12345:12345" + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + + dolphinscheduler-master: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "5678:5678" + - "5679:5679" + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] + + dolphinscheduler-worker: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + ports: + - "1234:1234" + - "1235:1235" + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: e2e: From ff515bb7e98de85d222a8915949500e7e1c3e984 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 08:30:31 +0800 Subject: [PATCH 060/104] fix code style --- .../test/resources/docker/cluster-test/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 7f60778997df..ca4e55ffcb3a 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,10 +31,10 @@ services: interval: 5s timeout: 60s retries: 120 - command: ["sh","-c","/opt/apache/bootstrap.sh"] + command: [ "sh","-c","/opt/apache/bootstrap.sh" ] networks: - e2e - dolphinscheduler-api: + dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 From 0fde9fbfb9cc9a64b1c86835fff338f28bb6ee54 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 09:00:19 +0800 Subject: [PATCH 061/104] 12 --- .../test/resources/docker/cluster-test/docker-compose.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index ca4e55ffcb3a..6d9165df2970 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -24,6 +24,8 @@ services: environment: ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + ports: + - "2181:2181" volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper healthcheck: @@ -31,9 +33,9 @@ services: interval: 5s timeout: 60s retries: 120 - command: [ "sh","-c","/opt/apache/bootstrap.sh" ] networks: - e2e + dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci environment: From 1f54b816d5b3fb5de07e6706567ad088a2e60dab Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 09:26:43 +0800 Subject: [PATCH 062/104] test --- .../resources/docker/cluster-test/docker-compose.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 6d9165df2970..00ef4701ec55 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -34,7 +34,7 @@ services: timeout: 60s retries: 120 networks: - - e2e + - dolphinscheduler dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci @@ -47,7 +47,7 @@ services: dolphinscheduler-zookeeper: condition: service_healthy networks: - - e2e + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s @@ -67,7 +67,7 @@ services: dolphinscheduler-zookeeper: condition: service_healthy networks: - - e2e + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 5s @@ -87,7 +87,7 @@ services: dolphinscheduler-zookeeper: condition: service_healthy networks: - - e2e + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 5s @@ -95,4 +95,4 @@ services: retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: - e2e: + dolphinscheduler: From e5ba60e6d68bf5701a15fdd001e654ebf521a035 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 09:41:57 +0800 Subject: [PATCH 063/104] fix docker-compose.yaml --- .../resources/docker/cluster-test/docker-compose.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 00ef4701ec55..3d318fe848ca 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -94,5 +94,15 @@ services: timeout: 5s retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] + networks: dolphinscheduler: + driver: bridge + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-worker-data: + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: From 23e93ec74a070de4776e46b56d2688dbd779363a Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 10:49:03 +0800 Subject: [PATCH 064/104] test --- .../resources/docker/cluster-test/docker-compose.yaml | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 3d318fe848ca..b6d279a1f358 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -95,14 +95,9 @@ services: retries: 120 command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +volumes: + dolphinscheduler-zookeeper: + networks: dolphinscheduler: driver: bridge - -volumes: - dolphinscheduler-postgresql: - dolphinscheduler-zookeeper: - dolphinscheduler-worker-data: - dolphinscheduler-logs: - dolphinscheduler-shared-local: - dolphinscheduler-resource-local: From 122d9396e754c139841733ef4270725d32d2d1e3 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 11:44:29 +0800 Subject: [PATCH 065/104] test --- .../test/resources/docker/cluster-test/docker-compose.yaml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index b6d279a1f358..9ec842dd7054 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -28,13 +28,14 @@ services: - "2181:2181" volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper + networks: + - dolphinscheduler healthcheck: test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181" ] interval: 5s timeout: 60s retries: 120 - networks: - - dolphinscheduler + command: [ "/bin/bash", "/bitnami/zookeeper/bin/zkServer.sh" ] dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci From d2304dfe63ea2e6e333508e650be751ef3842814 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 14:21:04 +0800 Subject: [PATCH 066/104] fix docker-compose.yaml --- .../docker/cluster-test/docker-compose.yaml | 138 ++++++++++++------ 1 file changed, 90 insertions(+), 48 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 9ec842dd7054..22790f01041d 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -1,10 +1,10 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # @@ -13,92 +13,134 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -# version: "3.8" services: + dolphinscheduler-postgresql: + image: bitnami/postgresql:15.2.0 + ports: + - "5432:5432" + profiles: ["all", "schema"] + environment: + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + volumes: + - dolphinscheduler-postgresql:/bitnami/postgresql + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - dolphinscheduler + dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 - profiles: [ "all" ] + profiles: ["all"] environment: ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons - ports: - - "2181:2181" volumes: - dolphinscheduler-zookeeper:/bitnami/zookeeper - networks: - - dolphinscheduler healthcheck: - test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181" ] + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] interval: 5s timeout: 60s retries: 120 - command: [ "/bin/bash", "/bitnami/zookeeper/bin/zkServer.sh" ] + networks: + - dolphinscheduler dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci - environment: - MASTER_MAX_CPU_LOAD_AVG: 100 - WORKER_TENANT_AUTO_CREATE: 'true' ports: - "12345:12345" + - "25333:25333" + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 depends_on: dolphinscheduler-zookeeper: condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] networks: - dolphinscheduler - healthcheck: - test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] - interval: 5s - timeout: 5s - retries: 120 - command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] - dolphinscheduler-master: + dolphinscheduler-alert: image: apache/dolphinscheduler-standalone-server:ci - environment: - MASTER_MAX_CPU_LOAD_AVG: 100 - WORKER_TENANT_AUTO_CREATE: 'true' - ports: - - "5678:5678" - - "5679:5679" + profiles: ["all"] + env_file: .env + healthcheck: + test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] + interval: 30s + timeout: 5s + retries: 3 depends_on: dolphinscheduler-zookeeper: condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] networks: - dolphinscheduler + + dolphinscheduler-master: + image: apache/dolphinscheduler-standalone-server:ci + profiles: ["all"] + env_file: .env healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] - interval: 5s + interval: 30s timeout: 5s - retries: 120 - command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] - - dolphinscheduler-worker: - image: apache/dolphinscheduler-standalone-server:ci - environment: - MASTER_MAX_CPU_LOAD_AVG: 100 - WORKER_TENANT_AUTO_CREATE: 'true' - ports: - - "1234:1234" - - "1235:1235" + retries: 3 depends_on: dolphinscheduler-zookeeper: condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] networks: - dolphinscheduler + + dolphinscheduler-worker: + image: apache/dolphinscheduler-standalone-server:ci + profiles: ["all"] + env_file: .env healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] - interval: 5s + interval: 30s timeout: 5s - retries: 120 + retries: 3 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] - -volumes: - dolphinscheduler-zookeeper: + networks: + - dolphinscheduler networks: dolphinscheduler: driver: bridge + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-worker-data: + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: From 7f0aa94ba0a7ff1fcdc2264d61d271ee9a86fcfb Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 15:32:32 +0800 Subject: [PATCH 067/104] fix docker-compose.yaml --- .../src/test/resources/docker/cluster-test/docker-compose.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 22790f01041d..3c774efba0c9 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -38,12 +38,9 @@ services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 - profiles: ["all"] environment: ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons - volumes: - - dolphinscheduler-zookeeper:/bitnami/zookeeper healthcheck: test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] interval: 5s From da38cce2c03b3dd68e9dc7fb323c137ea1ef08be Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 16:05:40 +0800 Subject: [PATCH 068/104] remove DB --- .../docker/cluster-test/docker-compose.yaml | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 3c774efba0c9..dd2d1e94b150 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -17,24 +17,6 @@ version: "3.8" services: - dolphinscheduler-postgresql: - image: bitnami/postgresql:15.2.0 - ports: - - "5432:5432" - profiles: ["all", "schema"] - environment: - POSTGRESQL_USERNAME: root - POSTGRESQL_PASSWORD: root - POSTGRESQL_DATABASE: dolphinscheduler - volumes: - - dolphinscheduler-postgresql:/bitnami/postgresql - healthcheck: - test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432"] - interval: 5s - timeout: 60s - retries: 120 - networks: - - dolphinscheduler dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 From 578a685a864dc2a0076b7774249b3e0697fe3ed6 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 17:05:59 +0800 Subject: [PATCH 069/104] test --- .../docker/cluster-test/docker-compose.yaml | 36 ++++++++----------- 1 file changed, 14 insertions(+), 22 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index dd2d1e94b150..29964ae64c53 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -46,10 +46,10 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler +# volumes: +# - dolphinscheduler-logs:/opt/dolphinscheduler/logs +# - dolphinscheduler-shared-local:/opt/soft +# - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] networks: - dolphinscheduler @@ -66,8 +66,8 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs +# volumes: +# - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] networks: - dolphinscheduler @@ -84,9 +84,9 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy - volumes: - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft +# volumes: +# - dolphinscheduler-logs:/opt/dolphinscheduler/logs +# - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] networks: - dolphinscheduler @@ -103,11 +103,11 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy - volumes: - - dolphinscheduler-worker-data:/tmp/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler +# volumes: +# - dolphinscheduler-worker-data:/tmp/dolphinscheduler +# - dolphinscheduler-logs:/opt/dolphinscheduler/logs +# - dolphinscheduler-shared-local:/opt/soft +# - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: - dolphinscheduler @@ -115,11 +115,3 @@ services: networks: dolphinscheduler: driver: bridge - -volumes: - dolphinscheduler-postgresql: - dolphinscheduler-zookeeper: - dolphinscheduler-worker-data: - dolphinscheduler-logs: - dolphinscheduler-shared-local: - dolphinscheduler-resource-local: From f42ba7602fbf4092d94f15998e0b312644da3e7c Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 17:31:14 +0800 Subject: [PATCH 070/104] test --- .../docker/cluster-test/docker-compose.yaml | 64 ++++++++++++------- 1 file changed, 40 insertions(+), 24 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 29964ae64c53..ecfff9ff6c8c 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -20,9 +20,12 @@ services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 + profiles: ["all"] environment: ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper healthcheck: test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] interval: 5s @@ -33,11 +36,14 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' ports: - "12345:12345" - "25333:25333" - profiles: ["all"] - env_file: .env + networks: + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 30s @@ -46,18 +52,18 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy -# volumes: -# - dolphinscheduler-logs:/opt/dolphinscheduler/logs -# - dolphinscheduler-shared-local:/opt/soft -# - dolphinscheduler-resource-local:/dolphinscheduler + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] - networks: - - dolphinscheduler + dolphinscheduler-alert: image: apache/dolphinscheduler-standalone-server:ci - profiles: ["all"] - env_file: .env + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' healthcheck: test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] interval: 30s @@ -66,16 +72,17 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy -# volumes: -# - dolphinscheduler-logs:/opt/dolphinscheduler/logs + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] networks: - dolphinscheduler dolphinscheduler-master: image: apache/dolphinscheduler-standalone-server:ci - profiles: ["all"] - env_file: .env + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 30s @@ -84,17 +91,18 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy -# volumes: -# - dolphinscheduler-logs:/opt/dolphinscheduler/logs -# - dolphinscheduler-shared-local:/opt/soft + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] networks: - dolphinscheduler dolphinscheduler-worker: image: apache/dolphinscheduler-standalone-server:ci - profiles: ["all"] - env_file: .env + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 30s @@ -103,11 +111,11 @@ services: depends_on: dolphinscheduler-zookeeper: condition: service_healthy -# volumes: -# - dolphinscheduler-worker-data:/tmp/dolphinscheduler -# - dolphinscheduler-logs:/opt/dolphinscheduler/logs -# - dolphinscheduler-shared-local:/opt/soft -# - dolphinscheduler-resource-local:/dolphinscheduler + volumes: + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: - dolphinscheduler @@ -115,3 +123,11 @@ services: networks: dolphinscheduler: driver: bridge + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-worker-data: + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: From e51ed50e6aa6001b34970cb1d74f2b9aa1961efc Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 17:42:09 +0800 Subject: [PATCH 071/104] test --- .../src/test/resources/docker/cluster-test/docker-compose.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index ecfff9ff6c8c..ad0cdb0683df 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -20,7 +20,6 @@ services: dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 - profiles: ["all"] environment: ALLOW_ANONYMOUS_LOGIN: "yes" ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons @@ -58,7 +57,6 @@ services: - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] - dolphinscheduler-alert: image: apache/dolphinscheduler-standalone-server:ci environment: From 21c74dfac6744356534f8dad0eaae70c2b3ae79b Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 18:14:35 +0800 Subject: [PATCH 072/104] test --- .../docker/cluster-test/docker-compose.yaml | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index ad0cdb0683df..c66b41e127b3 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -17,7 +17,6 @@ version: "3.8" services: - dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 environment: @@ -45,9 +44,9 @@ services: - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] - interval: 30s + interval: 5s timeout: 5s - retries: 3 + retries: 120 depends_on: dolphinscheduler-zookeeper: condition: service_healthy @@ -62,30 +61,32 @@ services: environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' + networks: + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] - interval: 30s + interval: 5s timeout: 5s - retries: 3 + retries: 120 depends_on: dolphinscheduler-zookeeper: condition: service_healthy volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] - networks: - - dolphinscheduler dolphinscheduler-master: image: apache/dolphinscheduler-standalone-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' + networks: + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] - interval: 30s + interval: 5s timeout: 5s - retries: 3 + retries: 120 depends_on: dolphinscheduler-zookeeper: condition: service_healthy @@ -93,19 +94,19 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] - networks: - - dolphinscheduler dolphinscheduler-worker: image: apache/dolphinscheduler-standalone-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' + networks: + - dolphinscheduler healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] - interval: 30s + interval: 5s timeout: 5s - retries: 3 + retries: 120 depends_on: dolphinscheduler-zookeeper: condition: service_healthy @@ -115,12 +116,9 @@ services: - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] - networks: - - dolphinscheduler networks: dolphinscheduler: - driver: bridge volumes: dolphinscheduler-postgresql: From 12402e25ca2759fb578a6a4da32f7a83c66d66b0 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 18:22:43 +0800 Subject: [PATCH 073/104] test --- .../resources/docker/cluster-test/docker-compose.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index c66b41e127b3..e5c818d1895f 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -49,7 +49,7 @@ services: retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_healthy + condition: service_completed_successfully volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft @@ -70,7 +70,7 @@ services: retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_healthy + condition: service_completed_successfully volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] @@ -89,7 +89,7 @@ services: retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_healthy + condition: service_completed_successfully volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft @@ -109,7 +109,7 @@ services: retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_healthy + condition: service_completed_successfully volumes: - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs From da6d710cfa639151bb97012abce7e44b0f80afb1 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 24 Oct 2024 19:42:16 +0800 Subject: [PATCH 074/104] test --- .../resources/docker/cluster-test/docker-compose.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index e5c818d1895f..e59c633baa73 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -46,7 +46,7 @@ services: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 depends_on: dolphinscheduler-zookeeper: condition: service_completed_successfully @@ -67,7 +67,7 @@ services: test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 depends_on: dolphinscheduler-zookeeper: condition: service_completed_successfully @@ -86,7 +86,7 @@ services: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 depends_on: dolphinscheduler-zookeeper: condition: service_completed_successfully @@ -106,7 +106,7 @@ services: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 depends_on: dolphinscheduler-zookeeper: condition: service_completed_successfully From d882305b0f969b0039d30c88cf8526bbb133c82f Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Fri, 25 Oct 2024 09:42:25 +0800 Subject: [PATCH 075/104] test --- .../src/test/resources/docker/cluster-test/docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index e59c633baa73..07d7f9ede503 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,6 +31,7 @@ services: retries: 120 networks: - dolphinscheduler + command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci From d0679264f1f4b83e3dc6b541a570a7ad1dae512a Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Fri, 25 Oct 2024 10:07:45 +0800 Subject: [PATCH 076/104] test12 --- .../docker/cluster-test/docker-compose.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 07d7f9ede503..bc3cf8e6ff3d 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,10 +31,10 @@ services: retries: 120 networks: - dolphinscheduler - command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] +# command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-api:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -55,10 +55,10 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler - command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-alert-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -74,10 +74,10 @@ services: condition: service_completed_successfully volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-master:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -94,10 +94,10 @@ services: volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-worker:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -116,7 +116,7 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler - command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: dolphinscheduler: From 5e17c186c4fa9baa69bffb0b69f5e51c0c5cfaaa Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Fri, 25 Oct 2024 10:50:38 +0800 Subject: [PATCH 077/104] test --- .github/workflows/e2e.yml | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1f73d46ff153..6d13194312cb 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,13 +87,16 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar + components=("master" "worker" "api" "tools" "alert-server" "standalone-server") + for component in "${components[@]}"; do + docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ + && du -sh /tmp/${component}-image.tar + done - uses: actions/upload-artifact@v4 name: Upload Docker Images with: name: standalone-image-e2e - path: /tmp/standalone-image.tar + path: /tmp/*.tar retention-days: 1 e2e: name: ${{ matrix.case.name }} @@ -173,7 +176,7 @@ jobs: path: /tmp - name: Load Docker Images run: | - docker load -i /tmp/standalone-image.tar + docker load -i /tmp/*.tar - name: Run Test run: | ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ From 1a807cff45d1990553e34603729b21180230db7b Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Fri, 25 Oct 2024 17:20:18 +0800 Subject: [PATCH 078/104] 12 --- .github/workflows/e2e.yml | 15 ++++---- .../resources/docker/cluster-test/Dockerfile | 34 +++++++++++++++++++ .../docker/cluster-test/docker-compose.yaml | 22 +++++++++--- 3 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6d13194312cb..44a5a7b72317 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,16 +87,13 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server" "standalone-server") - for component in "${components[@]}"; do - docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ - && du -sh /tmp/${component}-image.tar - done + docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ + && du -sh /tmp/standalone-image.tar && ll apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz && ll /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - uses: actions/upload-artifact@v4 name: Upload Docker Images with: name: standalone-image-e2e - path: /tmp/*.tar + path: /tmp/standalone-image.tar retention-days: 1 e2e: name: ${{ matrix.case.name }} @@ -128,8 +125,8 @@ jobs: class: org.apache.dolphinscheduler.e2e.cases.WorkflowHttpTaskE2ETest - name: WorkflowJavaTaskE2ETest class: org.apache.dolphinscheduler.e2e.cases.WorkflowJavaTaskE2ETest -# - name: WorkflowForSwitch -# class: org.apache.dolphinscheduler.e2e.cases.WorkflowSwitchE2ETest + # - name: WorkflowForSwitch + # class: org.apache.dolphinscheduler.e2e.cases.WorkflowSwitchE2ETest - name: FileManageE2ETest class: org.apache.dolphinscheduler.e2e.cases.FileManageE2ETest - name: MysqlDataSourceE2ETest @@ -176,7 +173,7 @@ jobs: path: /tmp - name: Load Docker Images run: | - docker load -i /tmp/*.tar + docker load -i /tmp/standalone-image.tar - name: Run Test run: | ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile new file mode 100644 index 000000000000..21f9a4d70249 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +FROM eclipse-temurin:8-jdk + +ENV DOCKER=true +ENV TZ=Asia/Shanghai +ENV DOLPHINSCHEDULER_HOME=/opt/dolphinscheduler + +RUN apt update ; \ + apt install -y sudo ; \ + rm -rf /var/lib/apt/lists/* + +WORKDIR $DOLPHINSCHEDULER_HOME + +COPY apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ + rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ + mv apache-dolphinscheduler-*-bin/* . ; \ + rm -rf apache-dolphinscheduler-*-bin diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index bc3cf8e6ff3d..83e828f34890 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,10 +31,13 @@ services: retries: 120 networks: - dolphinscheduler -# command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] + command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: image: apache/dolphinscheduler-api:ci + build: + context: . + dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -55,10 +58,13 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler -# command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: image: apache/dolphinscheduler-alert-server:ci + build: + context: . + dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -74,10 +80,13 @@ services: condition: service_completed_successfully volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs -# command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] + command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: image: apache/dolphinscheduler-master:ci + build: + context: . + dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -94,10 +103,13 @@ services: volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft -# command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] + command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: image: apache/dolphinscheduler-worker:ci + build: + context: . + dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -116,7 +128,7 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler -# command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] + command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: dolphinscheduler: From 6b9d33512ecb28d9bc8d17fb7b1df95cecd68598 Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Fri, 25 Oct 2024 18:38:01 +0800 Subject: [PATCH 079/104] Update e2e.yml --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 44a5a7b72317..11d90b0272c5 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,8 +87,8 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar && ll apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz && ll /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone to-image.tar \ + && du -sh /tmp/standalone-image.tar && ls apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz && ls /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - uses: actions/upload-artifact@v4 name: Upload Docker Images with: From 290b96dc5bea145bc0d4dd0132b067f4f36ef126 Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Sat, 26 Oct 2024 15:37:21 +0800 Subject: [PATCH 080/104] Update e2e.yml --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 11d90b0272c5..c1016666c10b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,7 +87,7 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone to-image.tar \ + docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standaloneto-image.tar \ && du -sh /tmp/standalone-image.tar && ls apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz && ls /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - uses: actions/upload-artifact@v4 name: Upload Docker Images From 138f457aa43ff25e2cdbf788cdd9a4841c5fba32 Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Sat, 26 Oct 2024 18:29:09 +0800 Subject: [PATCH 081/104] Update e2e.yml --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index c1016666c10b..127feb528007 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,7 +87,7 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standaloneto-image.tar \ + docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ && du -sh /tmp/standalone-image.tar && ls apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz && ls /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - uses: actions/upload-artifact@v4 name: Upload Docker Images From 49100a9c02f1edfe9d40820dde6a4591c90b6d9a Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Sat, 26 Oct 2024 19:30:18 +0800 Subject: [PATCH 082/104] Update e2e.yml --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 127feb528007..1f135e7587fd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -88,7 +88,7 @@ jobs: - name: Export Docker Images run: | docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar && ls apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz && ls /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + && du -sh /tmp/standalone-image.tar && ls ./ && ls /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - uses: actions/upload-artifact@v4 name: Upload Docker Images with: From e1053d96dc16fd2584b27044ea654ecd03824bc1 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 26 Oct 2024 22:34:05 +0800 Subject: [PATCH 083/104] test --- .github/workflows/e2e.yml | 2 +- .../src/test/resources/docker/cluster-test/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 1f135e7587fd..145230271bb4 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -88,7 +88,7 @@ jobs: - name: Export Docker Images run: | docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar && ls ./ && ls /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + && du -sh /tmp/standalone-image.tar && cp /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /tmp/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - uses: actions/upload-artifact@v4 name: Upload Docker Images with: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index 21f9a4d70249..67ae41197402 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -27,7 +27,7 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +COPY /tmp/dolphinscheduler/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ From b68eb0f434bc0f799c0b59c95f5c940785633cba Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 26 Oct 2024 22:52:35 +0800 Subject: [PATCH 084/104] test --- .github/workflows/e2e.yml | 13 ++++++++++++- .../test/resources/docker/cluster-test/Dockerfile | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 145230271bb4..b38d1e06316a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -88,13 +88,19 @@ jobs: - name: Export Docker Images run: | docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar && cp /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz /tmp/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + && du -sh /tmp/standalone-image.tar - uses: actions/upload-artifact@v4 name: Upload Docker Images with: name: standalone-image-e2e path: /tmp/standalone-image.tar retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload dist file + with: + name: bin-tar + path: /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz + retention-days: 1 e2e: name: ${{ matrix.case.name }} needs: build @@ -171,6 +177,11 @@ jobs: with: name: standalone-image-e2e path: /tmp + - uses: actions/download-artifact@v4 + name: Download dist file + with: + name: bin-tar + path: /tmp - name: Load Docker Images run: | docker load -i /tmp/standalone-image.tar diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile index 67ae41197402..d4a418bb4176 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/Dockerfile @@ -27,7 +27,7 @@ RUN apt update ; \ WORKDIR $DOLPHINSCHEDULER_HOME -COPY /tmp/dolphinscheduler/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME +COPY /tmp/apache-dolphinscheduler-*-bin.tar.gz $DOLPHINSCHEDULER_HOME RUN tar -zxvf apache-dolphinscheduler-*-bin.tar.gz ; \ rm -rf apache-dolphinscheduler-*-bin.tar.gz ; \ mv apache-dolphinscheduler-*-bin/* . ; \ From cef5839af753aaf39e492e6a38f9dcc58de770cc Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Sat, 26 Oct 2024 23:17:39 +0800 Subject: [PATCH 085/104] test --- .../test/resources/docker/cluster-test/docker-compose.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 83e828f34890..a6b92057858f 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -55,6 +55,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: + - /tmp:/tmp - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler @@ -79,6 +80,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: + - /tmp:/tmp - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] @@ -101,6 +103,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: + - /tmp:/tmp - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] @@ -124,6 +127,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: + - /tmp:/tmp - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft From 191e7e8eb319e1ee6de136aae7517a3c987184bc Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 09:51:48 +0800 Subject: [PATCH 086/104] test12 --- .github/workflows/e2e.yml | 3 +++ .../docker/cluster-test/docker-compose.yaml | 20 +++++++++++-------- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index b38d1e06316a..542786dfde7b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -182,6 +182,9 @@ jobs: with: name: bin-tar path: /tmp + - name: untar dist file + run: + tar -zxvf apache-dolphinscheduler-*-bin.tar.gz - name: Load Docker Images run: | docker load -i /tmp/standalone-image.tar diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index a6b92057858f..c9bdc2783c60 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -34,13 +34,14 @@ services: command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: - image: apache/dolphinscheduler-api:ci + image: apache/dolphinscheduler-standalone-server:ci build: context: . dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/api-server ports: - "12345:12345" - "25333:25333" @@ -55,20 +56,21 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp:/tmp + - /tmp/apache-dolphinscheduler-*-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: - image: apache/dolphinscheduler-alert-server:ci + image: apache/dolphinscheduler-standalone-server:ci build: context: . dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/alert-server networks: - dolphinscheduler healthcheck: @@ -80,18 +82,19 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp:/tmp + - /tmp/apache-dolphinscheduler-*-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: - image: apache/dolphinscheduler-master:ci + image: apache/dolphinscheduler-standalone-server:ci build: context: . dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/master-server networks: - dolphinscheduler healthcheck: @@ -103,19 +106,20 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp:/tmp + - /tmp/apache-dolphinscheduler-*-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: - image: apache/dolphinscheduler-worker:ci + image: apache/dolphinscheduler-standalone-server:ci build: context: . dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/worker-server networks: - dolphinscheduler healthcheck: @@ -127,7 +131,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp:/tmp + - /tmp/apache-dolphinscheduler-*-bin:/opt/dolphinscheduler - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft From f738337a8518691f137c60f2657740f19e602da5 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 10:14:05 +0800 Subject: [PATCH 087/104] fix --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 542786dfde7b..009eab0a8a62 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -184,7 +184,7 @@ jobs: path: /tmp - name: untar dist file run: - tar -zxvf apache-dolphinscheduler-*-bin.tar.gz + tar -zxvf /tmp/apache-dolphinscheduler-*-bin.tar.gz - name: Load Docker Images run: | docker load -i /tmp/standalone-image.tar From 207482acbb8085ca75f21520629cd87b5ed70eea Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 10:56:02 +0800 Subject: [PATCH 088/104] test123 --- .../docker/cluster-test/docker-compose.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index c9bdc2783c60..54a4ea19a502 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -35,9 +35,6 @@ services: dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci - build: - context: . - dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -64,9 +61,6 @@ services: dolphinscheduler-alert: image: apache/dolphinscheduler-standalone-server:ci - build: - context: . - dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -88,9 +82,6 @@ services: dolphinscheduler-master: image: apache/dolphinscheduler-standalone-server:ci - build: - context: . - dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' @@ -113,9 +104,6 @@ services: dolphinscheduler-worker: image: apache/dolphinscheduler-standalone-server:ci - build: - context: . - dockerfile: ./Dockerfile environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' From 7f8ade75bc03cf410ffe8c97f05d76946bb6d9fe Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 12:12:43 +0800 Subject: [PATCH 089/104] test1 --- .../resources/docker/cluster-test/docker-compose.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 54a4ea19a502..1c0caaef6d8b 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -53,7 +53,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp/apache-dolphinscheduler-*-bin:/opt/dolphinscheduler + - /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler @@ -76,7 +76,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp/apache-dolphinscheduler-*-bin:/opt/dolphinscheduler + - /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] @@ -97,7 +97,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp/apache-dolphinscheduler-*-bin:/opt/dolphinscheduler + - /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] @@ -119,7 +119,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp/apache-dolphinscheduler-*-bin:/opt/dolphinscheduler + - /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft From b9e21961ba792f4218ab1a4a05ae3b71b9590425 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 12:39:32 +0800 Subject: [PATCH 090/104] test123 --- .github/workflows/e2e.yml | 2 +- .../resources/docker/cluster-test/docker-compose.yaml | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 009eab0a8a62..4bc33406b32b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -184,7 +184,7 @@ jobs: path: /tmp - name: untar dist file run: - tar -zxvf /tmp/apache-dolphinscheduler-*-bin.tar.gz + tar -zxvf /tmp/apache-dolphinscheduler-*-bin.tar.gz && pwd - name: Load Docker Images run: | docker load -i /tmp/standalone-image.tar diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 1c0caaef6d8b..e5b1db136acf 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -53,7 +53,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - ./apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler @@ -76,7 +76,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - ./apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] @@ -97,7 +97,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - ./apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] @@ -119,7 +119,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /tmp/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - ./apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft From ae1455639c8ed77718096bd67502224ddda88518 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 13:54:27 +0800 Subject: [PATCH 091/104] test123 --- .../resources/docker/cluster-test/docker-compose.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index e5b1db136acf..6b6ead1c34c0 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -53,7 +53,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - ./apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler @@ -76,7 +76,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - ./apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] @@ -97,7 +97,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - ./apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] @@ -119,7 +119,7 @@ services: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - ./apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft From c62a5a2eb80bd4abd47669279e0ed9d313ad3be8 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 14:25:08 +0800 Subject: [PATCH 092/104] test23 --- .../docker/cluster-test/docker-compose.yaml | 51 +++++++------------ 1 file changed, 18 insertions(+), 33 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 6b6ead1c34c0..a210eca1cfe2 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -30,7 +30,7 @@ services: timeout: 60s retries: 120 networks: - - dolphinscheduler + - e2e command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: @@ -43,20 +43,17 @@ services: - "12345:12345" - "25333:25333" networks: - - dolphinscheduler + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] interval: 5s timeout: 5s - retries: 10 + retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_completed_successfully + condition: service_healthy volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: @@ -66,18 +63,17 @@ services: WORKER_TENANT_AUTO_CREATE: 'true' DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/alert-server networks: - - dolphinscheduler + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] interval: 5s timeout: 5s - retries: 10 + retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_completed_successfully + condition: service_healthy volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: @@ -87,19 +83,17 @@ services: WORKER_TENANT_AUTO_CREATE: 'true' DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/master-server networks: - - dolphinscheduler + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] interval: 5s timeout: 5s - retries: 10 + retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_completed_successfully + condition: service_healthy volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: @@ -109,30 +103,21 @@ services: WORKER_TENANT_AUTO_CREATE: 'true' DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/worker-server networks: - - dolphinscheduler + - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] interval: 5s timeout: 5s - retries: 10 + retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_completed_successfully + condition: service_healthy volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - - dolphinscheduler-worker-data:/tmp/dolphinscheduler - - dolphinscheduler-logs:/opt/dolphinscheduler/logs - - dolphinscheduler-shared-local:/opt/soft - - dolphinscheduler-resource-local:/dolphinscheduler + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: - dolphinscheduler: + e2e: volumes: - dolphinscheduler-postgresql: dolphinscheduler-zookeeper: - dolphinscheduler-worker-data: - dolphinscheduler-logs: - dolphinscheduler-shared-local: - dolphinscheduler-resource-local: From 1db53c4a38eda51b124766676c83e6af129e521d Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 17:49:53 +0800 Subject: [PATCH 093/104] test1 --- .../docker/cluster-test/docker-compose.yaml | 26 ++++++++++++++----- 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index a210eca1cfe2..bec219b0a318 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,7 +31,6 @@ services: retries: 120 networks: - e2e - command: [ "/bin/bash", "/opt/bitnami/zookeeper/bin/zkServer.sh"] dolphinscheduler-api: image: apache/dolphinscheduler-standalone-server:ci @@ -45,7 +44,7 @@ services: networks: - e2e healthcheck: - test: [ "CMD", "curl", "http://localhost:12345/dolphinscheduler/actuator/health" ] + test: [ "CMD", "curl", "http://127.0.0.1:12345/dolphinscheduler/actuator/health" ] interval: 5s timeout: 5s retries: 120 @@ -54,6 +53,9 @@ services: condition: service_healthy volumes: - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: @@ -65,7 +67,7 @@ services: networks: - e2e healthcheck: - test: [ "CMD", "curl", "http://localhost:50053/actuator/health" ] + test: [ "CMD", "curl", "http://127.0.0.1:50053/actuator/health" ] interval: 5s timeout: 5s retries: 120 @@ -74,6 +76,7 @@ services: condition: service_healthy volumes: - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: @@ -85,7 +88,7 @@ services: networks: - e2e healthcheck: - test: [ "CMD", "curl", "http://localhost:5679/actuator/health" ] + test: [ "CMD", "curl", "http://127.0.0.1:5679/actuator/health" ] interval: 5s timeout: 5s retries: 120 @@ -94,6 +97,8 @@ services: condition: service_healthy volumes: - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: @@ -105,19 +110,28 @@ services: networks: - e2e healthcheck: - test: [ "CMD", "curl", "http://localhost:1235/actuator/health" ] + test: [ "CMD", "curl", "http://127.0.0.1:1235/actuator/health" ] interval: 5s timeout: 5s retries: 120 depends_on: dolphinscheduler-zookeeper: - condition: service_healthy + condition: service_completed_successfully volumes: - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: e2e: volumes: + dolphinscheduler-postgresql: dolphinscheduler-zookeeper: + dolphinscheduler-worker-data: + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: From 2096564a99772e1d081bcdc1ec4761d9cef51371 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 18:34:27 +0800 Subject: [PATCH 094/104] 123 --- .../docker/cluster-test/docker-compose.yaml | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index bec219b0a318..561487509b9b 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -17,6 +17,36 @@ version: "3.8" services: + dolphinscheduler-postgresql: + image: bitnami/postgresql:15.2.0 + ports: + - "5432:5432" + environment: + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + volumes: + - dolphinscheduler-postgresql:/bitnami/postgresql + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432" ] + interval: 5s + timeout: 60s + retries: 120 + networks: + - e2e + dolphinscheduler-schema-initializer: + image: ${HUB}/dolphinscheduler-tools:${TAG} + command: [ tools/bin/upgrade-schema.sh ] + depends_on: + dolphinscheduler-postgresql: + condition: service_healthy + volumes: + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + networks: + - e2e + dolphinscheduler-zookeeper: image: bitnami/zookeeper:3.7.1 environment: From 36fada1a3775ad23e477b28d0abe9b32f223fea6 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 19:30:11 +0800 Subject: [PATCH 095/104] test123 --- .../test/resources/docker/cluster-test/docker-compose.yaml | 6 +++++- .../e2e/core/DolphinSchedulerExtension.java | 5 ++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 561487509b9b..604d061a4cf2 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -34,13 +34,17 @@ services: retries: 120 networks: - e2e + dolphinscheduler-schema-initializer: - image: ${HUB}/dolphinscheduler-tools:${TAG} + image: apache/dolphinscheduler-standalone-server:ci command: [ tools/bin/upgrade-schema.sh ] + environment: + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/tools depends_on: dolphinscheduler-postgresql: condition: service_healthy volumes: + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java index 351363154e91..89cfc8d45523 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java @@ -130,7 +130,7 @@ private void runInLocal() { private void runInDockerContainer(ExtensionContext context) { compose = createDockerCompose(context); compose.start(); - + compose.getContainerByServiceName(serviceName).get().getLogs(); address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort(serviceName, DOCKER_PORT)); rootPath = "/dolphinscheduler/ui/"; @@ -226,9 +226,8 @@ private ComposeContainer createDockerCompose(ExtensionContext context) { .withExposedService( serviceName, DOCKER_PORT, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300))) - .withLogConsumer(serviceName, outputFrame -> log.info(outputFrame.getUtf8String())) + .withLogConsumer(serviceName, outputFrame -> log.debug(outputFrame.getUtf8String())) .waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300))); - return compose; } } From 30abd03d3daed24f96ab8010af3db14f784be696 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Mon, 28 Oct 2024 19:53:05 +0800 Subject: [PATCH 096/104] test2231 --- .github/workflows/e2e.yml | 74 +++++++- .../docker/cluster-test/docker-compose.yaml | 33 ++-- .../docker/cluster-test/docker-compose12.yaml | 171 ++++++++++++++++++ 3 files changed, 252 insertions(+), 26 deletions(-) create mode 100644 dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose12.yaml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 4bc33406b32b..6aa725d56d48 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,14 +87,47 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - docker save apache/dolphinscheduler-standalone-server:ci -o /tmp/standalone-image.tar \ - && du -sh /tmp/standalone-image.tar + components=("master" "worker" "api" "tools" "alert-server" "standalone") + for component in "${components[@]}"; do + docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ + && du -sh /tmp/${component}-image.tar + done - uses: actions/upload-artifact@v4 name: Upload Docker Images with: name: standalone-image-e2e path: /tmp/standalone-image.tar retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload master Docker Images + with: + name: master-image-e2e + path: /tmp/master-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload api Docker Images + with: + name: api-image-e2e + path: /tmp/api-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload tools Docker Images + with: + name: tools-image-e2e + path: /tmp/tools-image.tar + retention-days: 1 + - uses: actions/upload-artifact@v4 + name: Upload alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server-image.tar + retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload dist file with: @@ -176,7 +209,37 @@ jobs: name: Download Docker Images with: name: standalone-image-e2e - path: /tmp + path: /tmp/standalone + - uses: actions/download-artifact@v4 + name: download master Docker Images + with: + name: master-image-e2e + path: /tmp/master + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download worker Docker Images + with: + name: worker-image-e2e + path: /tmp/worker + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download api Docker Images + with: + name: api-image-e2e + path: /tmp/api + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download tools Docker Images + with: + name: tools-image-e2e + path: /tmp/tools + retention-days: 1 + - uses: actions/download-artifact@v4 + name: download alert-server Docker Images + with: + name: alert-server-image-e2e + path: /tmp/alert-server + retention-days: 1 - uses: actions/download-artifact@v4 name: Download dist file with: @@ -187,7 +250,10 @@ jobs: tar -zxvf /tmp/apache-dolphinscheduler-*-bin.tar.gz && pwd - name: Load Docker Images run: | - docker load -i /tmp/standalone-image.tar + components=("master" "worker" "api" "tools" "alert-server") + for component in "${components[@]}"; do + docker load -i /tmp/${component}/${component}-image.tar + done - name: Run Test run: | ./mvnw -B -f dolphinscheduler-e2e/pom.xml -am \ diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 604d061a4cf2..8623bd906ee5 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -31,20 +31,17 @@ services: test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432" ] interval: 5s timeout: 60s - retries: 120 + retries: 10 networks: - e2e dolphinscheduler-schema-initializer: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-tools:ci command: [ tools/bin/upgrade-schema.sh ] - environment: - DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/tools depends_on: dolphinscheduler-postgresql: condition: service_healthy volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler @@ -62,16 +59,15 @@ services: test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] interval: 5s timeout: 60s - retries: 120 + retries: 10 networks: - e2e dolphinscheduler-api: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-api:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/api-server ports: - "12345:12345" - "25333:25333" @@ -81,78 +77,71 @@ services: test: [ "CMD", "curl", "http://127.0.0.1:12345/dolphinscheduler/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 depends_on: dolphinscheduler-zookeeper: condition: service_healthy volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-alert-server:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/alert-server networks: - e2e healthcheck: test: [ "CMD", "curl", "http://127.0.0.1:50053/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 depends_on: dolphinscheduler-zookeeper: condition: service_healthy volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-master:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/master-server networks: - e2e healthcheck: test: [ "CMD", "curl", "http://127.0.0.1:5679/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 depends_on: dolphinscheduler-zookeeper: condition: service_healthy volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: - image: apache/dolphinscheduler-standalone-server:ci + image: apache/dolphinscheduler-worker:ci environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/worker-server networks: - e2e healthcheck: test: [ "CMD", "curl", "http://127.0.0.1:1235/actuator/health" ] interval: 5s timeout: 5s - retries: 120 + retries: 10 depends_on: dolphinscheduler-zookeeper: condition: service_completed_successfully volumes: - - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler - dolphinscheduler-worker-data:/tmp/dolphinscheduler - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose12.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose12.yaml new file mode 100644 index 000000000000..604d061a4cf2 --- /dev/null +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose12.yaml @@ -0,0 +1,171 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: "3.8" + +services: + dolphinscheduler-postgresql: + image: bitnami/postgresql:15.2.0 + ports: + - "5432:5432" + environment: + POSTGRESQL_USERNAME: root + POSTGRESQL_PASSWORD: root + POSTGRESQL_DATABASE: dolphinscheduler + volumes: + - dolphinscheduler-postgresql:/bitnami/postgresql + healthcheck: + test: [ "CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/5432" ] + interval: 5s + timeout: 60s + retries: 120 + networks: + - e2e + + dolphinscheduler-schema-initializer: + image: apache/dolphinscheduler-standalone-server:ci + command: [ tools/bin/upgrade-schema.sh ] + environment: + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/tools + depends_on: + dolphinscheduler-postgresql: + condition: service_healthy + volumes: + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + networks: + - e2e + + dolphinscheduler-zookeeper: + image: bitnami/zookeeper:3.7.1 + environment: + ALLOW_ANONYMOUS_LOGIN: "yes" + ZOO_4LW_COMMANDS_WHITELIST: srvr,ruok,wchs,cons + volumes: + - dolphinscheduler-zookeeper:/bitnami/zookeeper + healthcheck: + test: ["CMD", "bash", "-c", "cat < /dev/null > /dev/tcp/127.0.0.1/2181"] + interval: 5s + timeout: 60s + retries: 120 + networks: + - e2e + + dolphinscheduler-api: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/api-server + ports: + - "12345:12345" + - "25333:25333" + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://127.0.0.1:12345/dolphinscheduler/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] + + dolphinscheduler-alert: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/alert-server + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://127.0.0.1:50053/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] + + dolphinscheduler-master: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/master-server + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://127.0.0.1:5679/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + dolphinscheduler-zookeeper: + condition: service_healthy + volumes: + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] + + dolphinscheduler-worker: + image: apache/dolphinscheduler-standalone-server:ci + environment: + MASTER_MAX_CPU_LOAD_AVG: 100 + WORKER_TENANT_AUTO_CREATE: 'true' + DOLPHINSCHEDULER_HOME: /opt/dolphinscheduler/worker-server + networks: + - e2e + healthcheck: + test: [ "CMD", "curl", "http://127.0.0.1:1235/actuator/health" ] + interval: 5s + timeout: 5s + retries: 120 + depends_on: + dolphinscheduler-zookeeper: + condition: service_completed_successfully + volumes: + - /home/runner/work/dolphinscheduler/dolphinscheduler/apache-dolphinscheduler-dev-SNAPSHOT-bin:/opt/dolphinscheduler + - dolphinscheduler-worker-data:/tmp/dolphinscheduler + - dolphinscheduler-logs:/opt/dolphinscheduler/logs + - dolphinscheduler-shared-local:/opt/soft + - dolphinscheduler-resource-local:/dolphinscheduler + command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] + +networks: + e2e: + +volumes: + dolphinscheduler-postgresql: + dolphinscheduler-zookeeper: + dolphinscheduler-worker-data: + dolphinscheduler-logs: + dolphinscheduler-shared-local: + dolphinscheduler-resource-local: From f87fe90a5744b641116ef4218e81f6157e03b87f Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Mon, 28 Oct 2024 21:30:56 +0800 Subject: [PATCH 097/104] Update e2e.yml --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6aa725d56d48..a8c0c2720cda 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -87,7 +87,7 @@ jobs: -Pdocker,staging -Ddocker.tag=ci - name: Export Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server" "standalone") + components=("master" "worker" "api" "tools" "alert-server" "standalone-server") for component in "${components[@]}"; do docker save apache/dolphinscheduler-${component}:ci -o /tmp/${component}-image.tar \ && du -sh /tmp/${component}-image.tar @@ -250,7 +250,7 @@ jobs: tar -zxvf /tmp/apache-dolphinscheduler-*-bin.tar.gz && pwd - name: Load Docker Images run: | - components=("master" "worker" "api" "tools" "alert-server") + components=("master" "worker" "api" "tools" "alert-server" "standalone-server") for component in "${components[@]}"; do docker load -i /tmp/${component}/${component}-image.tar done From 61be7e19f65f4f13da7ed26cef656c7b329c1c0e Mon Sep 17 00:00:00 2001 From: wangxj3 <857234426@qq.com> Date: Mon, 28 Oct 2024 21:49:35 +0800 Subject: [PATCH 098/104] Update e2e.yml --- .github/workflows/e2e.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index a8c0c2720cda..324232fae095 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -96,7 +96,7 @@ jobs: name: Upload Docker Images with: name: standalone-image-e2e - path: /tmp/standalone-image.tar + path: /tmp/standalone-server-image.tar retention-days: 1 - uses: actions/upload-artifact@v4 name: Upload master Docker Images @@ -209,7 +209,7 @@ jobs: name: Download Docker Images with: name: standalone-image-e2e - path: /tmp/standalone + path: /tmp/standalone-server - uses: actions/download-artifact@v4 name: download master Docker Images with: From ad79b46f392356d2a4a24fbc335fbb1b6861790c Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Tue, 29 Oct 2024 11:07:30 +0800 Subject: [PATCH 099/104] timeout --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 324232fae095..20b5f59f4f2c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -51,7 +51,7 @@ jobs: needs: paths-filter if: ${{ (needs.paths-filter.outputs.not-ignore == 'true') || (github.event_name == 'push') }} runs-on: ubuntu-latest - timeout-minutes: 20 + timeout-minutes: 30 steps: - uses: actions/checkout@v4 with: From 1c847d46efeb8eb2875699aaffe6bc9325f97f25 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 31 Oct 2024 14:10:40 +0800 Subject: [PATCH 100/104] test123 --- .github/workflows/e2e.yml | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 20b5f59f4f2c..fd370e0d2fa0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -128,12 +128,6 @@ jobs: name: alert-server-image-e2e path: /tmp/alert-server-image.tar retention-days: 1 - - uses: actions/upload-artifact@v4 - name: Upload dist file - with: - name: bin-tar - path: /home/runner/work/dolphinscheduler/dolphinscheduler/dolphinscheduler-dist/target/apache-dolphinscheduler-dev-SNAPSHOT-bin.tar.gz - retention-days: 1 e2e: name: ${{ matrix.case.name }} needs: build @@ -240,14 +234,6 @@ jobs: name: alert-server-image-e2e path: /tmp/alert-server retention-days: 1 - - uses: actions/download-artifact@v4 - name: Download dist file - with: - name: bin-tar - path: /tmp - - name: untar dist file - run: - tar -zxvf /tmp/apache-dolphinscheduler-*-bin.tar.gz && pwd - name: Load Docker Images run: | components=("master" "worker" "api" "tools" "alert-server" "standalone-server") From 0215c38e583d2d98b83fa043c1b9740eb29b3fbd Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 31 Oct 2024 14:44:22 +0800 Subject: [PATCH 101/104] test123 --- .github/workflows/e2e.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index fd370e0d2fa0..eccb5c7fc91b 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -181,6 +181,22 @@ jobs: env: RECORDING_PATH: /tmp/recording-${{ matrix.case.name }} steps: + - uses: actions/checkout@v4 + with: + submodules: true + - name: Maximize runner space + uses: ./.github/actions/maximize-build-space + with: + root-reserve-mb: 30720 + temp-reserve-mb: 10240 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' + remove-codeql: 'true' + remove-docker-images: 'true' + - uses: actions/checkout@v4 + with: + submodules: true - uses: actions/checkout@v4 with: submodules: true From b05c27aec0e49681723e6d1769c5c4b5ad7a1531 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 31 Oct 2024 15:23:40 +0800 Subject: [PATCH 102/104] 1234 --- .../resources/docker/cluster-test/docker-compose.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 8623bd906ee5..2feeabb3b06d 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -37,7 +37,7 @@ services: dolphinscheduler-schema-initializer: image: apache/dolphinscheduler-tools:ci - command: [ tools/bin/upgrade-schema.sh ] + command: [ bin/upgrade-schema.sh ] depends_on: dolphinscheduler-postgresql: condition: service_healthy @@ -85,7 +85,7 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler - command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/api-server/bin/start.sh" ] dolphinscheduler-alert: image: apache/dolphinscheduler-alert-server:ci @@ -104,7 +104,7 @@ services: condition: service_healthy volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/alert-server/bin/start.sh" ] dolphinscheduler-master: image: apache/dolphinscheduler-master:ci @@ -124,7 +124,7 @@ services: volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/master-server/bin/start.sh" ] dolphinscheduler-worker: image: apache/dolphinscheduler-worker:ci @@ -146,7 +146,7 @@ services: - dolphinscheduler-logs:/opt/dolphinscheduler/logs - dolphinscheduler-shared-local:/opt/soft - dolphinscheduler-resource-local:/dolphinscheduler - command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] +# command: [ "/bin/bash", "/opt/dolphinscheduler/worker-server/bin/start.sh" ] networks: e2e: From 256a20e7063291f4c47b1542703ac172bfd9a96e Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 31 Oct 2024 15:24:49 +0800 Subject: [PATCH 103/104] 123 --- .../test/resources/docker/cluster-test/docker-compose.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 2feeabb3b06d..643237aafd9d 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -79,7 +79,7 @@ services: timeout: 5s retries: 10 depends_on: - dolphinscheduler-zookeeper: + dolphinscheduler-schema-initializer: condition: service_healthy volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs @@ -119,7 +119,7 @@ services: timeout: 5s retries: 10 depends_on: - dolphinscheduler-zookeeper: + dolphinscheduler-schema-initializer: condition: service_healthy volumes: - dolphinscheduler-logs:/opt/dolphinscheduler/logs From 1f4df9e475576093fedf9551f62fe1a063abade4 Mon Sep 17 00:00:00 2001 From: wangxj959 Date: Thu, 31 Oct 2024 15:56:42 +0800 Subject: [PATCH 104/104] 234 --- .../test/resources/docker/cluster-test/docker-compose.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml index 643237aafd9d..53c62eb167d6 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/cluster-test/docker-compose.yaml @@ -37,7 +37,9 @@ services: dolphinscheduler-schema-initializer: image: apache/dolphinscheduler-tools:ci - command: [ bin/upgrade-schema.sh ] + env_file: .env + profiles: [ "schema" ] + command: [ tools/bin/upgrade-schema.sh ] depends_on: dolphinscheduler-postgresql: condition: service_healthy