Cert rotator

scripts/performance/common.py

@@ -18,6 +18,7 @@
 import os
 import sys
 import time
+import base64
 from typing import Callable, List, Optional, Tuple, Type, TypeVar
 
 
@@ -139,6 +140,10 @@ def get_packages_directory() -> str:
     '''
     return os.path.join(get_artifacts_directory(), 'packages')
 
+def base64_to_bytes(base64_string: str) -> bytes:
+    byte_data = base64.b64decode(base64_string)
+    return byte_data
+
 @contextmanager
 def push_dir(path: Optional[str] = None):
     '''
@@ -234,6 +239,7 @@ def __init__(
             cmdline: List[str],
             success_exit_codes: Optional[List[int]] = None,
             verbose: bool = False,
+            echo: bool = True,
             retry: int = 0):
         if cmdline is None:
             raise TypeError('Unspecified command line to be executed.')
@@ -243,6 +249,7 @@ def __init__(
         self.__cmdline = cmdline
         self.__verbose = verbose
         self.__retry = retry
+        self.__echo = echo
 
         if success_exit_codes is None:
             self.__success_exit_codes = [0]
@@ -262,6 +269,11 @@ def success_exit_codes(self) -> List[int]:
         '''
         return self.__success_exit_codes
 
+    @property
+    def echo(self) -> bool:
+        '''Enables/Disables echoing of STDOUT'''
+        return self.__echo
+
     @property
     def verbose(self) -> bool:
         '''Enables/Disables verbosity.'''
@@ -297,7 +309,8 @@ def __runinternal(self, working_directory: Optional[str] = None) -> Tuple[int, s
                             line = raw_line.decode('utf-8', errors='backslashreplace')
                             self.__stdout.write(line)
                             line = line.rstrip()
-                            getLogger().info(line)
+                            if self.echo:
+                                getLogger().info(line)
                 proc.wait()
                 return (proc.returncode, quoted_cmdline)
 

scripts/performance/constants.py

@@ -5,4 +5,5 @@
 UPLOAD_STORAGE_URI = 'https://pvscmdupload.{}.core.windows.net'
 UPLOAD_QUEUE = 'resultsqueue'
 TENANT_ID = '72f988bf-86f1-41af-91ab-2d7cd011db47'
-CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4'
+ARC_CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4'
+CERT_CLIENT_ID = '8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc'

scripts/run_performance_job.py

@@ -774,6 +774,32 @@ def run_performance_job(args: RunPerformanceJobArgs):
     getLogger().info("Copying global.json to payload directory")
     shutil.copy(os.path.join(args.performance_repo_dir, 'global.json'), os.path.join(performance_payload_dir, 'global.json'))
 
+    # Building CertHelper needs to happen here as we need it on every run. This also means that we will need to move the calculation
+    # of the parameters needed outside of the if block
+
+    framework = os.environ["PERFLAB_Framework"]
+    os.environ["PERFLAB_TARGET_FRAMEWORKS"] = framework
+    if args.os_group == "windows":
+        runtime_id = f"win-{args.architecture}"
+    elif args.os_group == "osx":
+        runtime_id = f"osx-{args.architecture}"
+    else:
+        runtime_id = f"linux-{args.architecture}"
+
+    dotnet_executable_path = os.path.join(ci_setup_arguments.install_dir, "dotnet")
+
+    RunCommand([
+        dotnet_executable_path, "publish", 
+        "-c", "Release", 
+        "-o", os.path.join(payload_dir, "certhelper"),
+        "-f", framework,
+        "-r", runtime_id,
+        "--self-contained",
+        os.path.join(args.performance_repo_dir, "src", "tools", "CertHelper", "CertHelper.csproj"),
+        f"/bl:{os.path.join(args.performance_repo_dir, 'artifacts', 'log', build_config, 'CertHelper.binlog')}",
+        "-p:DisableTransitiveFrameworkReferenceDownloads=true"],
+        verbose=True).run()
+
     if args.is_scenario:
         set_environment_variable("DOTNET_ROOT", ci_setup_arguments.install_dir, save_to_pipeline=True)
         getLogger().info(f"Set DOTNET_ROOT to {ci_setup_arguments.install_dir}")
@@ -782,17 +808,6 @@ def run_performance_job(args: RunPerformanceJobArgs):
         set_environment_variable("PATH", new_path, save_to_pipeline=True)
         getLogger().info(f"Set PATH to {new_path}")
 
-        framework = os.environ["PERFLAB_Framework"]
-        os.environ["PERFLAB_TARGET_FRAMEWORKS"] = framework
-        if args.os_group == "windows":
-            runtime_id = f"win-{args.architecture}"
-        elif args.os_group == "osx":
-            runtime_id = f"osx-{args.architecture}"
-        else:
-            runtime_id = f"linux-{args.architecture}"
-
-        dotnet_executable_path = os.path.join(ci_setup_arguments.install_dir, "dotnet")
-
         os.environ["MSBUILDDISABLENODEREUSE"] = "1" # without this, MSbuild will be kept alive
 
         # build Startup

scripts/upload.py

@@ -3,11 +3,11 @@
 from azure.storage.blob import BlobClient, ContentSettings
 from azure.storage.queue import QueueClient, TextBase64EncodePolicy
 from azure.core.exceptions import ResourceExistsError, ClientAuthenticationError
-from azure.identity import DefaultAzureCredential, ClientAssertionCredential
+from azure.identity import DefaultAzureCredential, ClientAssertionCredential, CertificateCredential
 from traceback import format_exc
 from glob import glob
-from performance.common import retry_on_exception
-from performance.constants import TENANT_ID, CLIENT_ID
+from performance.common import retry_on_exception, RunCommand, helixpayload, base64_to_bytes, extension
+from performance.constants import TENANT_ID, ARC_CLIENT_ID, CERT_CLIENT_ID
 import os
 import json
 
@@ -32,14 +32,25 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag
         credential = None
         try:
             dac = DefaultAzureCredential()
-            credential = ClientAssertionCredential(TENANT_ID, CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token)
+            credential = ClientAssertionCredential(TENANT_ID, ARC_CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token)
             credential.get_token("https://storage.azure.com/.default")
         except ClientAuthenticationError as ex:
-            getLogger().info("Unable to use managed identity. Falling back to environment variable.")
-            credential = os.getenv(sas_token_env)
+            credential = None
+            getLogger().info("Unable to use managed identity. Falling back to certificate.")
+            cmd_line = [(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))]
+            cert_helper = RunCommand(cmd_line, None, True, False, 0)
+            cert_helper.run()
+            for cert in cert_helper.stdout.splitlines():
+                credential = CertificateCredential(TENANT_ID, CERT_CLIENT_ID, certificate_data=base64_to_bytes(cert))
+                try:
+                    credential.get_token("https://storage.azure.com/.default")
+                except ClientAuthenticationError as ex:
+                    credential = None
+                    continue
         if credential is None:
-            getLogger().error("Sas token environment variable {} was not defined.".format(sas_token_env))
-            return 1
+            getLogger().error("Unable to authenticate with managed identity or certificates.")
+            getLogger().info("Falling back to environment variable.")
+            credential = os.getenv(sas_token_env)
 
         files = glob(globpath, recursive=True)
         any_upload_or_queue_failed = False

src/tools/CertHelper/AssemblyInfo.cs

@@ -0,0 +1,12 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Text;
+using System.Threading.Tasks;
+
+[assembly: InternalsVisibleTo("CertHelperTests")]
+namespace CertHelper;
+internal class AssemblyInfo
+{
+}

src/tools/CertHelper/CertHelper.csproj

@@ -0,0 +1,20 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+      <TargetFramework>$(PERFLAB_TARGET_FRAMEWORKS)</TargetFramework>
+      <!-- Supported target frameworks -->
+      <TargetFramework Condition="'$(TargetFramework)' == ''">net9.0</TargetFramework>
+    <ImplicitUsings>enable</ImplicitUsings>
+    <Nullable>enable</Nullable>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <PackageReference Include="Azure.Core" Version="1.44.1" />
+    <PackageReference Include="Azure.Identity" Version="1.11.4" />
+    <PackageReference Include="Azure.Security.KeyVault.Certificates" Version="4.7.0" />
+    <PackageReference Include="Azure.Security.KeyVault.Secrets" Version="4.7.0" />
+    <PackageReference Include="Azure.Storage.Blobs" Version="12.23.0" />
+  </ItemGroup>
+
+</Project>

src/tools/CertHelper/CertHelper.sln

@@ -0,0 +1,28 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+# Visual Studio Version 17
+VisualStudioVersion = 17.12.35514.174 d17.12
+MinimumVisualStudioVersion = 10.0.40219.1
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertHelper", "CertHelper.csproj", "{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertRotatorTests", "..\CertHelperTests\CertRotatorTests.csproj", "{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.Build.0 = Release|Any CPU
+		{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(SolutionProperties) = preSolution
+		HideSolutionNode = FALSE
+	EndGlobalSection
+EndGlobal

src/tools/CertHelper/Constants.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CertHelper;
+public class Constants
+{
+    public static readonly string Cert1Name = "LabCert1";
+    public static readonly string Cert2Name = "LabCert2";
+    public static readonly Uri Cert1Id = new Uri("https://test.vault.azure.net/certificates/LabCert1/07a7d98bf4884e5c40e690e02b96b3b4");
+    public static readonly Uri Cert2Id = new Uri("https://test.vault.azure.net/certificates/LabCert2/07a7d98bf4884e5c41e690e02b96b3b4");
+}

src/tools/CertHelper/IX509Store.cs

@@ -0,0 +1,35 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CertHelper;
+public interface IX509Store
+{
+    X509Certificate2Collection Certificates { get; }
+    string? Name { get; }
+    StoreLocation Location { get; }
+    X509Store GetX509Store();
+}
+
+public class TestableX509Store : IX509Store
+{
+    public X509Certificate2Collection Certificates { get => store.Certificates; }
+
+    public string? Name => store.Name;
+
+    public StoreLocation Location => store.Location;
+
+    private X509Store store;
+    public TestableX509Store(OpenFlags flags = OpenFlags.ReadOnly)
+    {
+        store = new X509Store(StoreName.My, StoreLocation.CurrentUser, flags);
+    }
+
+    public X509Store GetX509Store()
+    {
+        return store;
+    }
+}

src/tools/CertHelper/KeyVaultCert.cs

@@ -0,0 +1,110 @@
+using Azure;
+using Azure.Core;
+using Azure.Identity;
+using Azure.Security.KeyVault.Certificates;
+using Azure.Security.KeyVault.Secrets;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Runtime.CompilerServices;
+using System.Security.Cryptography.X509Certificates;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace CertHelper;
+
+public class KeyVaultCert
+{
+    private readonly string _keyVaultUrl = "https://dotnetperfkeyvault.vault.azure.net/";
+    private readonly string _tenantId = "72f988bf-86f1-41af-91ab-2d7cd011db47";
+    private readonly string _clientId = "8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc";
+
+    public X509Certificate2Collection KeyVaultCertificates { get; set; }
+    public ILocalCert LocalCerts { get; set; }
+    private TokenCredential _credential { get; set; }
+    private CertificateClient _certClient { get; set; }
+    private SecretClient _secretClient { get; set; }
+
+    public KeyVaultCert(TokenCredential? cred = null, CertificateClient? certClient = null, SecretClient? secretClient = null, ILocalCert? localCerts = null)
+    {
+        LocalCerts = localCerts ?? new LocalCert();
+        _credential = cred ?? GetCertifcateCredentialAsync(_tenantId, _clientId, LocalCerts.Certificates).Result;
+        _certClient = certClient ?? new CertificateClient(new Uri(_keyVaultUrl), _credential);
+        _secretClient = secretClient ?? new SecretClient(new Uri(_keyVaultUrl), _credential);
+        KeyVaultCertificates = new X509Certificate2Collection();
+    }
+
+    public async Task LoadKeyVaultCertsAsync()
+    {
+        KeyVaultCertificates.Add(await FindCertificateInKeyVaultAsync(Constants.Cert1Name));
+        KeyVaultCertificates.Add(await FindCertificateInKeyVaultAsync(Constants.Cert2Name));
+
+        if (KeyVaultCertificates.Where(c => c == null).Count() > 0)
+        {
+            throw new Exception("One or more certificates not found");
+        }
+    }
+
+    private async Task<ClientCertificateCredential> GetCertifcateCredentialAsync(string tenantId, string clientId, X509Certificate2Collection certCollection)
+    {
+        ClientCertificateCredential? ccc = null;
+        Exception? exception = null;
+        foreach (var cert in certCollection)
+        {
+            try
+            {
+                ccc = new ClientCertificateCredential(tenantId, clientId, cert);
+                await ccc.GetTokenAsync(new TokenRequestContext(new string[] { "https://vault.azure.net/.default" }));
+                break;
+            }
+            catch (Exception ex)
+            {
+                ccc = null;
+                exception = ex;
+            }
+        }
+        if(ccc == null)
+        {
+            throw new Exception("Both certificates failed to authenticate", exception);
+        }
+        return ccc;
+    }
+
+    private async Task<X509Certificate2> FindCertificateInKeyVaultAsync(string certName)
+    {
+        var keyVaultCert = await _certClient.GetCertificateAsync(certName);
+        if(keyVaultCert.Value == null)
+        {
+            throw new Exception("Certificate not found in Key Vault");
+        }
+        var secret = await _secretClient.GetSecretAsync(keyVaultCert.Value.Name, keyVaultCert.Value.SecretId.Segments.Last());
+        if(secret.Value == null)
+        {
+            throw new Exception("Certificate secret not found in Key Vault");
+        }
+        var certBytes = Convert.FromBase64String(secret.Value.Value);
+#if NET9_0_OR_GREATER        
+        var cert = X509CertificateLoader.LoadPkcs12(certBytes, "", X509KeyStorageFlags.Exportable);
+#else
+        var cert = new X509Certificate2(certBytes, "", X509KeyStorageFlags.Exportable);
+#endif
+        return cert;
+    }
+
+    public bool ShouldRotateCerts()
+    {
+        var keyVaultThumbprints = new HashSet<string>();
+        foreach (var cert in KeyVaultCertificates)
+        {
+            keyVaultThumbprints.Add(cert.Thumbprint);
+        }
+        foreach(var cert in LocalCerts.Certificates)
+        {
+            if (!keyVaultThumbprints.Contains(cert.Thumbprint))
+            {
+                return true;
+            }
+        }
+        return false;
+    }
+}