Merge pull request #395 from serverlessworkflow/feat-secrets

Add secrets support

Author: cdavernas

.dockerignore

@@ -21,5 +21,4 @@
 **/obj
 **/secrets.dev.yaml
 **/values.dev.yaml
-LICENSE
-README.md
+LICENSE

README.md

@@ -1,5 +1,5 @@
-<p align="center">
-  <img src="assets/images/transparent_logo.png" height="350px" alt="Synapse Logo"/>
+<p align="center">
+  <img src="./assets/images/transparent_logo.png" height="350px" alt="Synapse Logo"/>
 </p>
 
 ---
@@ -19,6 +19,10 @@ It enables developers and organizations to define and execute workflows effortle
 
 With Synapse, you can create powerful workflows that are cloud vendor-agnostic, easily scalable, and highly customizable.
 
+<p align="center">
+  <img src="./assets/images/preview.gif" alt="Synapse Preview"/>
+</p>
+
 ### Features
 
 - **Easy to Use**: The Serverless Workflow DSL is designed for universal understanding, enabling users to quickly grasp workflow concepts and create complex workflows effortlessly.
@@ -33,7 +37,7 @@ With Synapse, you can create powerful workflows that are cloud vendor-agnostic,
 - **Scalable**: Promotes code reusability, maintainability, and scalability across different environments.
 - **Cross-Platform**: Runs on various operating systems, providing flexibility and ease of integration.
 
-### Microservices
+### Architecture
 
 Synapse is composed of several specialized applications, allowing for atomic scalability, resilience, and ease of maintenance:
 
@@ -43,6 +47,12 @@ Synapse is composed of several specialized applications, allowing for atomic sca
 - **Correlator**: Performs Complex Event Processing (CEP) and correlates ingested events.
 - **CLI**: Allows interaction with the Synapse API via the command line interface.
 
+<p align="center">
+  <img src="./assets/images/architecture-c4-l2.png" alt="Synapse Architecture C4 Diagram - Container Layer"/>
+</p>
+
+*For more information about the Synapse architecture, please refer to the [wiki](https://github.com/serverlessworkflow/synapse/wiki/Architecture).* 📖
+
 ## Getting Started
 
 ### Prerequisites
@@ -79,6 +89,8 @@ The simplest way to get started is by using the provided Docker Compose setup.
 
 This will pull the necessary Docker images and start the Synapse services as defined in the `docker-compose.yml` file. You can then access the Synapse API and dashboard as configured.
 
+*For more information about installing Synapse, please refer to the [wiki](https://github.com/serverlessworkflow/synapse/wiki/Installation)*. 📖
+
 ### Run using `synctl` Command-line Interface
 
 First, set up the Synapse API server to use with `synctl`:
@@ -120,15 +132,15 @@ The command above will provide the fully qualified name of the created workflow
 synctl workflow-instance get-output greeter-uk58h3dssqp620a --namespace default --output yaml
 ```
 
-For more information about `synctl`, please refer to the [documentation](#synctl).
+*For more information about `synctl`, please refer to the [wiki](https://github.com/serverlessworkflow/synapse/wiki/CLI-Usage).* 📖
 
 ## Community
 
 The Synapse project has a vibrant and growing community dedicated to building a community-driven and vendor-neutral workflow runtime ecosystem. Contributions from the community are encouraged and essential to the continued growth and success of the project.
 
-A list of community members who have contributed to Synapse can be found [here](./community/README.md).
+A list of community members who have contributed to Synapse can be found [here](./community/README.md). 👥
 
-To learn how to contribute to Synapse, please refer to the [contribution guidelines](CONTRIBUTING.md).
+To learn how to contribute to Synapse, please refer to the [contribution guidelines](CONTRIBUTING.md). 📝
 
 For any copyright-related questions when contributing to a CNCF project like Synapse, please refer to the [Ownership of Copyrights in CNCF Project Contributions](https://github.com/cncf/foundation/blob/master/copyright-notices.md) document.
 
@@ -138,5 +150,5 @@ As contributors and maintainers of Synapse, and in the interest of fostering an
 
 The project is committed to making participation in Synapse a harassment-free experience for everyone, regardless of experience level, gender identity and expression, sexual orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or nationality.
 
-For more detailed information, please see the full project Code of Conduct [here](code-of-conduct.md).
+For more detailed information, please see the full project Code of Conduct [here](code-of-conduct.md). 🛡️
 

assets/images/architecture-c4-l2.png

@@ -29,6 +29,7 @@ services:
       SYNAPSE_RUNNER_API: http://api:8080
       SYNAPSE_RUNNER_LIFECYCLE_EVENTS: true
       SYNAPSE_RUNNER_CONTAINER_PLATFORM: docker
+      SYNAPSE_RUNTIME_DOCKER_SECRETS_DIRECTORY: C:\Users\User\.synapse\secrets
       DOCKER_HOST: unix:///var/run/docker.sock
     extra_hosts:
       - "host.docker.internal:host-gateway"

assets/images/preview.gif

@@ -6,6 +6,8 @@ EXPOSE 8080
 FROM mcr.microsoft.com/dotnet/sdk:8.0 AS build
 ARG BUILD_CONFIGURATION=Release
 WORKDIR /src
+COPY ["README.md", "README.md"]
+COPY ["assets/images/transparent_logomark_256.png", "assets/images/transparent_logomark_256.png"]
 COPY ["src/api/Synapse.Api.Server/Synapse.Api.Server.csproj", "src/api/Synapse.Api.Server/"]
 RUN dotnet restore "./src/api/Synapse.Api.Server/Synapse.Api.Server.csproj"
 COPY . .

deployments/docker-compose/docker-compose.yml

@@ -66,7 +66,7 @@ public virtual async Task<OAuth2Token> GetTokenAsync(OAuth2AuthenticationSchemeD
         {
             var discoveryRequest = new DiscoveryDocumentRequest()
             {
-                Address = configuration.Authority.OriginalString,
+                Address = configuration.Authority!.OriginalString,
                 Policy = new()
                 {
                     RequireHttps = false
@@ -80,7 +80,7 @@ public virtual async Task<OAuth2Token> GetTokenAsync(OAuth2AuthenticationSchemeD
         else throw new NotSupportedException($"The specified scheme type '{configuration.GetType().FullName}' is not supported in this context");
         var properties = new Dictionary<string, string>()
         {
-            { "grant_type", configuration.Grant }
+            { "grant_type", configuration.Grant! }
         };
         switch (configuration.Client?.Authentication)
         {

src/api/Synapse.Api.Server/Dockerfile

@@ -0,0 +1,43 @@
+// Copyright © 2024-Present The Synapse Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License"),
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+namespace Synapse;
+
+/// <summary>
+/// Defines extensions for <see cref="AuthenticationPolicyDefinition"/>s
+/// </summary>
+public static class AuthenticationPolicyDefinitionExtensions
+{
+
+    /// <summary>
+    /// Attempts to get the name of the secret, if any, on which the <see cref="AuthenticationPolicyDefinition"/> is based
+    /// </summary>
+    /// <param name="authentication">The extended <see cref="AuthenticationPolicyDefinition"/></param>
+    /// <param name="secretName">The name of the secret, if any, on which the <see cref="AuthenticationPolicyDefinition"/> is based</param>
+    /// <returns>A boolean indicating whether or not the <see cref="AuthenticationPolicyDefinition"/> is secret based</returns>
+    public static bool TryGetBaseSecret(this AuthenticationPolicyDefinition authentication, out string? secretName)
+    {
+        secretName = authentication.Scheme switch
+        {
+            AuthenticationScheme.Basic => authentication.Basic?.Use,
+            AuthenticationScheme.Bearer => authentication.Bearer?.Use,
+            AuthenticationScheme.Certificate => authentication.Certificate?.Use,
+            AuthenticationScheme.Digest => authentication.Digest?.Use,
+            AuthenticationScheme.OAuth2 => authentication.OAuth2?.Use,
+            AuthenticationScheme.OpenIDConnect => authentication.Oidc?.Use,
+            _ => throw new NotSupportedException($"The specified authentication schema '{authentication.Scheme}' is not supported")
+        };
+        return !string.IsNullOrWhiteSpace(secretName);
+    }
+
+}

src/core/Synapse.Core.Infrastructure/Services/OAuth2TokenManager.cs

@@ -35,17 +35,38 @@ public record DockerRuntimeConfiguration
         Image = SynapseDefaults.Containers.Images.Runner
     };
 
+    /// <summary>
+    /// Initializes a new <see cref="DockerRuntimeConfiguration"/>
+    /// </summary>
+    public DockerRuntimeConfiguration()
+    {
+        var env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Docker.Api.Endpoint);
+        if (!string.IsNullOrWhiteSpace(env) && Uri.TryCreate(env, UriKind.RelativeOrAbsolute, out var uri)) this.Api.Endpoint = uri;
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Docker.Api.Version);
+        if (!string.IsNullOrWhiteSpace(env)) this.Api.Version = env;
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Docker.Image.Registry);
+        if (!string.IsNullOrWhiteSpace(env)) this.ImageRegistry = env;
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Docker.Image.PullPolicy);
+        if (!string.IsNullOrWhiteSpace(env)) this.ImagePullPolicy = env;
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Docker.Secrets.Directory);
+        if (!string.IsNullOrWhiteSpace(env)) this.Secrets.Directory = env;
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Docker.Secrets.MountPath);
+        if (!string.IsNullOrWhiteSpace(env)) this.Secrets.MountPath = env;
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Docker.Network);
+        if (!string.IsNullOrWhiteSpace(env)) this.Network = env;
+    }
+
     /// <summary>
     /// Gets/sets the Docker API to use
     /// </summary>
     [DataMember(Order = 1, Name = "api"), JsonPropertyOrder(1), JsonPropertyName("api"), YamlMember(Order = 1, Alias = "api")]
     public virtual DockerApiConfiguration Api { get; set; } = new();
 
     /// <summary>
-    /// Gets/sets the name of the image repository to use when pulling the runtime's container image
+    /// Gets/sets the name of the image registry to use when pulling the runtime's container image
     /// </summary>
-    [DataMember(Order = 2, Name = "imageRepository"), JsonPropertyOrder(2), JsonPropertyName("imageRepository"), YamlMember(Order = 2, Alias = "imageRepository")]
-    public virtual string? ImageRepository { get; set; }
+    [DataMember(Order = 2, Name = "imageRegistry"), JsonPropertyOrder(2), JsonPropertyName("imageRegistry"), YamlMember(Order = 2, Alias = "imageRegistry")]
+    public virtual string? ImageRegistry { get; set; }
 
     /// <summary>
     /// Gets/sets the Docker image pull policy. Supported values are 'Always', 'IfNotPresent' and 'Never'. Defaults to 'Always'.

src/core/Synapse.Core/Extensions/AuthenticationPolicyDefinitionExtensions.cs

@@ -52,6 +52,19 @@ public record KubernetesRuntimeConfiguration
         }
     };
 
+    /// <summary>
+    /// Initializes a new <see cref="KubernetesRuntimeConfiguration"/>
+    /// </summary>
+    public KubernetesRuntimeConfiguration()
+    {
+        var env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Kubernetes.Kubeconfig);
+        if (!string.IsNullOrWhiteSpace(env)) this.Kubeconfig = env;
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Kubernetes.Secrets.VolumeName);
+        if (!string.IsNullOrWhiteSpace(env)) this.Secrets.VolumeName = env;
+        env = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Kubernetes.Secrets.MountPath);
+        if (!string.IsNullOrWhiteSpace(env)) this.Secrets.MountPath = env;
+    }
+
     /// <summary>
     /// Gets/sets the path to the Kubeconfig file to use, if any. If not set, defaults to 'InCluster' configuration
     /// </summary>
@@ -76,7 +89,7 @@ public record KubernetesRuntimeConfiguration
     /// <returns>The runner container template</returns>
     public static V1Pod LoadPodTemplate()
     {
-        var templateFilePath = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Docker.Container);
+        var templateFilePath = Environment.GetEnvironmentVariable(SynapseDefaults.EnvironmentVariables.Runtime.Kubernetes.Pod);
         if (string.IsNullOrWhiteSpace(templateFilePath) || !File.Exists(templateFilePath)) return DefaultPodTemplate;
         var yaml = File.ReadAllText(templateFilePath);
         return YamlSerializer.Default.Deserialize<V1Pod>(yaml)!;

src/core/Synapse.Core/Resources/DockerRuntimeConfiguration.cs

@@ -30,7 +30,7 @@ public record KubernetesRuntimeSecretsConfiguration
     public const string DefaultMountPath = "/run/secrets/synapse";
 
     /// <summary>
-    /// Gets/sets the name on which to mounts secrets
+    /// Gets/sets the name of the volume on which to mounts secrets
     /// </summary>
     [DataMember(Order = 1, Name = "volumeName"), JsonPropertyOrder(1), JsonPropertyName("volumeName"), YamlMember(Order = 1, Alias = "volumeName")]
     public virtual string VolumeName { get; set; } = DefaultVolumeName;

src/core/Synapse.Core/Resources/KubernetesRuntimeConfiguration.cs

@@ -585,10 +585,125 @@ public static class Docker
                 /// Gets the prefix for all Docker runtime related environment variables
                 /// </summary>
                 public const string Prefix = Runtime.Prefix + "DOCKER_";
+
                 /// <summary>
                 /// Gets the environment variable used to specify the YAML file used to configure the Docker runner container
                 /// </summary>
                 public const string Container = Prefix + "CONTAINER";
+                /// <summary>
+                /// Gets the environment variable used to configure the network runner containers should be connected to
+                /// </summary>
+                public const string Network = Prefix + "NETWORK";
+
+                /// <summary>
+                /// Exposes constants about environment variables used to configure the API of a Docker runtime
+                /// </summary>
+                public static class Api
+                {
+
+                    /// <summary>
+                    /// Gets the prefix for all Docker runtime API related environment variables
+                    /// </summary>
+                    public const string Prefix = Docker.Prefix + "API_";
+
+                    /// <summary>
+                    /// Gets the environment variable used to configure the endpoint of the Docker API to use
+                    /// </summary>
+                    public const string Endpoint = Prefix + "ENDPOINT";
+                    /// <summary>
+                    /// Gets the environment variable used to configure the version of the Docker API to use
+                    /// </summary>
+                    public const string Version = Prefix + "VERSION";
+
+                }
+
+                /// <summary>
+                /// Exposes constants about environment variables used to configure the runner images of a Docker runtime
+                /// </summary>
+                public static class Image
+                {
+
+                    /// <summary>
+                    /// Gets the prefix for all Docker runtime image related environment variables
+                    /// </summary>
+                    public const string Prefix = Docker.Prefix + "IMAGE_";
+
+                    /// <summary>
+                    /// Gets the environment variable used to configure the image registry to use when pulling runner images
+                    /// </summary>
+                    public const string Registry = Prefix + "REGISTRY";
+                    /// <summary>
+                    /// Gets the environment variable used to configure the policy to use when pulling runner images
+                    /// </summary>
+                    public const string PullPolicy = Prefix + "PULL_POLICY";
+
+                }
+
+                /// <summary>
+                /// Exposes constants about environment variables used to configure the secrets used by a Docker runtime
+                /// </summary>
+                public static class Secrets
+                {
+
+                    /// <summary>
+                    /// Gets the prefix for all Docker runtime secrets related environment variables
+                    /// </summary>
+                    public const string Prefix = Docker.Prefix + "SECRETS_";
+
+                    /// <summary>
+                    /// Gets the environment variable used to configure the directory that contains the secrets to mount onto runner containers
+                    /// </summary>
+                    public const string Directory = Prefix + "DIRECTORY";
+                    /// <summary>
+                    /// Gets the environment variable used to configure the directory to mount the secrets volume to
+                    /// </summary>
+                    public const string MountPath = Prefix + "MOUNT_PATH";
+
+                }
+
+            }
+
+            /// <summary>
+            /// Exposes constants about Kubernetes runtime-related environment variables
+            /// </summary>
+            public static class Kubernetes
+            {
+
+                /// <summary>
+                /// Gets the prefix for all Kubernetes runtime related environment variables
+                /// </summary>
+                public const string Prefix = Runtime.Prefix + "K8S_";
+
+                /// <summary>
+                /// Gets the environment variable used to configure the path to the Kubeconfig file to use
+                /// </summary>
+                public const string Kubeconfig = Prefix + "KUBECONFIG";
+                /// <summary>
+                /// Gets the environment variable used to specify the YAML file used to configure the Kubernetes runner pod
+                /// </summary>
+                public const string Pod = Prefix + "POD";
+
+                /// <summary>
+                /// Exposes constants about environment variables used to configure the secrets used by a Docker runtime
+                /// </summary>
+                public static class Secrets
+                {
+
+                    /// <summary>
+                    /// Gets the prefix for all Kubernetes runtime secrets related environment variables
+                    /// </summary>
+                    public const string Prefix = Kubernetes.Prefix + "SECRETS_";
+
+                    /// <summary>
+                    /// Gets the environment variable used to configure the name of the volume onto which to mount secrets
+                    /// </summary>
+                    public const string VolumeName = Prefix + "VOLUME_NAME";
+                    /// <summary>
+                    /// Gets the environment variable used to configure the directory to mount the secrets volume to
+                    /// </summary>
+                    public const string MountPath = Prefix + "MOUNT_PATH";
+
+                }
 
             }
 
@@ -672,7 +787,7 @@ public static class Images
             /// <summary>
             /// Gets the current version of Synapse container images
             /// </summary>
-            public static string Version = typeof(SynapseDefaults).Assembly.GetName().Version?.ToString(3) ?? "latest";
+            public static readonly string Version = typeof(SynapseDefaults).Assembly.GetName().Version?.ToString(3) ?? "latest";
             /// <summary>
             /// Gets the name of the Synapse API container image
             /// </summary>
@@ -688,7 +803,7 @@ public static class Images
             /// <summary>
             /// Gets the name of the Synapse Runner container image
             /// </summary>
-            public static readonly string Runner = $"{ImageRegistry}/runner:{Version}";
+            public static readonly string Runner = $"{ImageRegistry}/runner:latest"; //todo: $"{ImageRegistry}/runner:{Version}";
 
         }