From 395debaae53abfa64d86e07899c5b615a383c9dd Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 8 Nov 2024 15:29:38 -0800 Subject: [PATCH 01/24] WIP --- scripts/certdeploy.ps1 | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 scripts/certdeploy.ps1 diff --git a/scripts/certdeploy.ps1 b/scripts/certdeploy.ps1 new file mode 100644 index 00000000000..afaad49f391 --- /dev/null +++ b/scripts/certdeploy.ps1 @@ -0,0 +1,33 @@ +sleep 240 +# Download the certificates + +# Path to the certificate files +$certPath1 = "LabCert1.cer" +$certPath2 = "LabCert2.cer" + +# Friendly names for the certificates +$friendlyName1 = "Cert 1" +$friendlyName2 = "Cert 2" + +# Import the certificates +$cert1 = Import-Certificate -FilePath $certPath1 -CertStoreLocation Cert:\LocalMachine\My +$cert2 = Import-Certificate -FilePath $certPath2 -CertStoreLocation Cert:\LocalMachine\My + +# Set the friendly names +$cert1.FriendlyName = $friendlyName1 +$cert2.FriendlyName = $friendlyName2 + +# Define the action to run a PowerShell script +$action = New-ScheduledTaskAction -Execute 'C:\CertRotator\CertRotator.exe' + +# Define the trigger to run daily at 8 AM +$trigger = New-ScheduledTaskTrigger -Once -At "12:00AM" -RepetitionInterval (New-TimeSpan -Hours 1) -RepetitionDuration (New-TimeSpan -Days 3650) + +# Define the principal (user) under which the task will run +$principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest + +# Register the scheduled task +Register-ScheduledTask -Action $action -Trigger $trigger -Principal $principal -TaskName "CertRotatorTask" -Description "Runs the certificate rotator script every hour." + +mkdir C:\CertRotator +Copy-Item %HELIX_WORKITEM_ROOT%\CertRotator.exe C:\CertRotator \ No newline at end of file From e8034ad358ee8b23bb51f1b082d612ba8f096bf4 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 2 Jan 2025 16:39:54 -0800 Subject: [PATCH 02/24] Add certificate handling for auth --- scripts/performance/common.py | 3 +++ scripts/performance/constants.py | 4 +++- scripts/upload.py | 37 ++++++++++++++++++++++++++------ 3 files changed, 37 insertions(+), 7 deletions(-) diff --git a/scripts/performance/common.py b/scripts/performance/common.py index df58c05570b..87647c39921 100644 --- a/scripts/performance/common.py +++ b/scripts/performance/common.py @@ -38,6 +38,9 @@ def get_machine_architecture(): def iswin(): return sys.platform == 'win32' +def islinux(): + return sys.platform == 'linux' + def extension(): 'gets platform specific extension' return '.exe' if iswin() else '' diff --git a/scripts/performance/constants.py b/scripts/performance/constants.py index 02b63119988..77af75796e2 100644 --- a/scripts/performance/constants.py +++ b/scripts/performance/constants.py @@ -5,4 +5,6 @@ UPLOAD_STORAGE_URI = 'https://pvscmdupload.{}.core.windows.net' UPLOAD_QUEUE = 'resultsqueue' TENANT_ID = '72f988bf-86f1-41af-91ab-2d7cd011db47' -CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4' +ARC_CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4' +CERT_SUBJECT = 'CN=dotnetperf.microsoft.com' +CERT_CLIENT_ID = '8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc' \ No newline at end of file diff --git a/scripts/upload.py b/scripts/upload.py index 40d83e3156b..380adb44996 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -3,13 +3,15 @@ from azure.storage.blob import BlobClient, ContentSettings from azure.storage.queue import QueueClient, TextBase64EncodePolicy from azure.core.exceptions import ResourceExistsError, ClientAuthenticationError -from azure.identity import DefaultAzureCredential, ClientAssertionCredential +from azure.identity import DefaultAzureCredential, ClientAssertionCredential, CertificateCredential from traceback import format_exc from glob import glob -from performance.common import retry_on_exception -from performance.constants import TENANT_ID, CLIENT_ID +from performance.common import retry_on_exception, iswin, islinux +from performance.constants import TENANT_ID, ARC_CLIENT_ID, CERT_SUBJECT, CERT_CLIENT_ID import os import json +import ssl +from cryptography import x509 from logging import getLogger @@ -32,11 +34,34 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag credential = None try: dac = DefaultAzureCredential() - credential = ClientAssertionCredential(TENANT_ID, CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token) + credential = ClientAssertionCredential(TENANT_ID, ARC_CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token) credential.get_token("https://storage.azure.com/.default") except ClientAuthenticationError as ex: - getLogger().info("Unable to use managed identity. Falling back to environment variable.") - credential = os.getenv(sas_token_env) + getLogger().info("Unable to use managed identity. Falling back to certificate.") + cert_collection = list() + if iswin(): + for bin_cert in ssl.enum_certificates("MY"): + cert = x509.load_pem_x509_certificate(bin_cert[0]) + if cert and cert.subject == CERT_SUBJECT: + cert_collection.append(cert) + elif islinux(): + user_dir = os.path.expanduser("~") + cert_dir = os.path.join(user_dir, ".dotnet/corefx/cryptography/x509stores/my") + pfx_files = glob(os.path.join(cert_dir, "*.pfx")) + + for pfx_file in pfx_files: + with open(pfx_file, "rb") as f: + pfx_data = f.read() + cert = x509.load_pem_x509_certificate(pfx_data) + if cert and cert.subject == CERT_SUBJECT: + cert_collection.append(cert) + for cert in cert_collection: + try: + credential = CertificateCredential(TENANT_ID, CERT_CLIENT_ID, cert) + credential.get_token("https://storage.azure.com/.default") + break + except ClientAuthenticationError as ex: + getLogger().info("Certificate not valid for client id: {0}".format(ARC_CLIENT_ID)) if credential is None: getLogger().error("Sas token environment variable {} was not defined.".format(sas_token_env)) return 1 From b455366e9b39977e880de4aee3b9cd25e1bd8b80 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 7 Jan 2025 16:30:28 -0800 Subject: [PATCH 03/24] Add cert handling for upload.py --- scripts/performance/common.py | 5 +++++ scripts/upload.py | 36 ++++++++++++----------------------- 2 files changed, 17 insertions(+), 24 deletions(-) diff --git a/scripts/performance/common.py b/scripts/performance/common.py index 87647c39921..5fcfba6e038 100644 --- a/scripts/performance/common.py +++ b/scripts/performance/common.py @@ -18,6 +18,7 @@ import os import sys import time +import base64 from typing import Callable, List, Optional, Tuple, Type, TypeVar @@ -142,6 +143,10 @@ def get_packages_directory() -> str: ''' return os.path.join(get_artifacts_directory(), 'packages') +def base64_to_bytes(base64_string: str) -> bytes: + byte_data = base64.b64decode(base64_string) + return byte_data + @contextmanager def push_dir(path: Optional[str] = None): ''' diff --git a/scripts/upload.py b/scripts/upload.py index 380adb44996..3f98979db3f 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -6,7 +6,7 @@ from azure.identity import DefaultAzureCredential, ClientAssertionCredential, CertificateCredential from traceback import format_exc from glob import glob -from performance.common import retry_on_exception, iswin, islinux +from performance.common import retry_on_exception, iswin, islinux, RunCommand, helixpayload, base64_to_bytes from performance.constants import TENANT_ID, ARC_CLIENT_ID, CERT_SUBJECT, CERT_CLIENT_ID import os import json @@ -37,34 +37,22 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag credential = ClientAssertionCredential(TENANT_ID, ARC_CLIENT_ID, lambda: dac.get_token("api://AzureADTokenExchange/.default").token) credential.get_token("https://storage.azure.com/.default") except ClientAuthenticationError as ex: + credential = None getLogger().info("Unable to use managed identity. Falling back to certificate.") - cert_collection = list() - if iswin(): - for bin_cert in ssl.enum_certificates("MY"): - cert = x509.load_pem_x509_certificate(bin_cert[0]) - if cert and cert.subject == CERT_SUBJECT: - cert_collection.append(cert) - elif islinux(): - user_dir = os.path.expanduser("~") - cert_dir = os.path.join(user_dir, ".dotnet/corefx/cryptography/x509stores/my") - pfx_files = glob(os.path.join(cert_dir, "*.pfx")) - - for pfx_file in pfx_files: - with open(pfx_file, "rb") as f: - pfx_data = f.read() - cert = x509.load_pem_x509_certificate(pfx_data) - if cert and cert.subject == CERT_SUBJECT: - cert_collection.append(cert) - for cert in cert_collection: + cmd_line = list((os.path.join(str(helixpayload()), 'Microsoft.Dotnet.Performance.CertHelper.exe'))) + cert_helper = RunCommand(cmd_line, None, False, 0) + cert_helper.run() + for cert in cert_helper.stdout.splitlines(): + credential = CertificateCredential(TENANT_ID, CERT_CLIENT_ID, certificate_data=base64_to_bytes(cert)) try: - credential = CertificateCredential(TENANT_ID, CERT_CLIENT_ID, cert) credential.get_token("https://storage.azure.com/.default") - break except ClientAuthenticationError as ex: - getLogger().info("Certificate not valid for client id: {0}".format(ARC_CLIENT_ID)) + credential = None + continue if credential is None: - getLogger().error("Sas token environment variable {} was not defined.".format(sas_token_env)) - return 1 + getLogger().error("Unable to authenticate with managed identity or certificates.") + getLogger().info("Falling back to environment variable.") + credential = os.getenv(sas_token_env) files = glob(globpath, recursive=True) any_upload_or_queue_failed = False From 9ab2ef1591cbed631d574f4aa76a1094ba50bb1a Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 7 Jan 2025 16:32:26 -0800 Subject: [PATCH 04/24] Remove certdeploy --- scripts/certdeploy.ps1 | 33 --------------------------------- 1 file changed, 33 deletions(-) delete mode 100644 scripts/certdeploy.ps1 diff --git a/scripts/certdeploy.ps1 b/scripts/certdeploy.ps1 deleted file mode 100644 index afaad49f391..00000000000 --- a/scripts/certdeploy.ps1 +++ /dev/null @@ -1,33 +0,0 @@ -sleep 240 -# Download the certificates - -# Path to the certificate files -$certPath1 = "LabCert1.cer" -$certPath2 = "LabCert2.cer" - -# Friendly names for the certificates -$friendlyName1 = "Cert 1" -$friendlyName2 = "Cert 2" - -# Import the certificates -$cert1 = Import-Certificate -FilePath $certPath1 -CertStoreLocation Cert:\LocalMachine\My -$cert2 = Import-Certificate -FilePath $certPath2 -CertStoreLocation Cert:\LocalMachine\My - -# Set the friendly names -$cert1.FriendlyName = $friendlyName1 -$cert2.FriendlyName = $friendlyName2 - -# Define the action to run a PowerShell script -$action = New-ScheduledTaskAction -Execute 'C:\CertRotator\CertRotator.exe' - -# Define the trigger to run daily at 8 AM -$trigger = New-ScheduledTaskTrigger -Once -At "12:00AM" -RepetitionInterval (New-TimeSpan -Hours 1) -RepetitionDuration (New-TimeSpan -Days 3650) - -# Define the principal (user) under which the task will run -$principal = New-ScheduledTaskPrincipal -UserId "SYSTEM" -LogonType ServiceAccount -RunLevel Highest - -# Register the scheduled task -Register-ScheduledTask -Action $action -Trigger $trigger -Principal $principal -TaskName "CertRotatorTask" -Description "Runs the certificate rotator script every hour." - -mkdir C:\CertRotator -Copy-Item %HELIX_WORKITEM_ROOT%\CertRotator.exe C:\CertRotator \ No newline at end of file From 1d8371670f905ff0ef5c3e2076b57459ee51aec8 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 7 Jan 2025 16:33:26 -0800 Subject: [PATCH 05/24] Remove old constants --- scripts/performance/constants.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/performance/constants.py b/scripts/performance/constants.py index 77af75796e2..bd29d394404 100644 --- a/scripts/performance/constants.py +++ b/scripts/performance/constants.py @@ -6,5 +6,4 @@ UPLOAD_QUEUE = 'resultsqueue' TENANT_ID = '72f988bf-86f1-41af-91ab-2d7cd011db47' ARC_CLIENT_ID = 'a231f733-103b-46e9-b58a-9416edde0eb4' -CERT_SUBJECT = 'CN=dotnetperf.microsoft.com' CERT_CLIENT_ID = '8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc' \ No newline at end of file From c6ff45fd105f61a1079d9c8797e105c545cdf035 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 7 Jan 2025 16:44:08 -0800 Subject: [PATCH 06/24] Add CertHelper code and build step --- scripts/run_performance_job.py | 13 +++++++++++++ scripts/upload.py | 2 +- src/tools/CertHelper/CertHelper.csproj | 10 ++++++++++ src/tools/CertHelper/Program.cs | 22 ++++++++++++++++++++++ 4 files changed, 46 insertions(+), 1 deletion(-) create mode 100644 src/tools/CertHelper/CertHelper.csproj create mode 100644 src/tools/CertHelper/Program.cs diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 9ff3684825f..9214f447717 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -807,6 +807,19 @@ def run_performance_job(args: RunPerformanceJobArgs): f"/bl:{os.path.join(args.performance_repo_dir, 'artifacts', 'log', build_config, 'Startup.binlog')}", "-p:DisableTransitiveFrameworkReferenceDownloads=true"], verbose=True).run() + + # build CertHelper + RunCommand([ + dotnet_executable_path, "publish", + "-c", "Release", + "-o", os.path.join(payload_dir, "certhelper"), + "-f", framework, + "-r", runtime_id, + "--self-contained", + os.path.join(args.performance_repo_dir, "src", "tools", "CertHelper", "CertHelper.csproj"), + f"/bl:{os.path.join(args.performance_repo_dir, 'artifacts', 'log', build_config, 'CertHelper.binlog')}", + "-p:DisableTransitiveFrameworkReferenceDownloads=true"], + verbose=True).run() # build SizeOnDisk RunCommand([ diff --git a/scripts/upload.py b/scripts/upload.py index 3f98979db3f..3c64020f54a 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -7,7 +7,7 @@ from traceback import format_exc from glob import glob from performance.common import retry_on_exception, iswin, islinux, RunCommand, helixpayload, base64_to_bytes -from performance.constants import TENANT_ID, ARC_CLIENT_ID, CERT_SUBJECT, CERT_CLIENT_ID +from performance.constants import TENANT_ID, ARC_CLIENT_ID, CERT_CLIENT_ID import os import json import ssl diff --git a/src/tools/CertHelper/CertHelper.csproj b/src/tools/CertHelper/CertHelper.csproj new file mode 100644 index 00000000000..2150e3797ba --- /dev/null +++ b/src/tools/CertHelper/CertHelper.csproj @@ -0,0 +1,10 @@ + + + + Exe + net8.0 + enable + enable + + + diff --git a/src/tools/CertHelper/Program.cs b/src/tools/CertHelper/Program.cs new file mode 100644 index 00000000000..1b0384f4615 --- /dev/null +++ b/src/tools/CertHelper/Program.cs @@ -0,0 +1,22 @@ +using System.Security.Cryptography.X509Certificates; +using System.Text; + +namespace CertHelper +{ + internal class Program + { + static void Main(string[] args) + { + using(var store = new X509Store(StoreName.My, StoreLocation.CurrentUser, OpenFlags.ReadWrite)) + { + foreach(var cert in store.Certificates) + { + if (cert.Subject.Contains("CN=dotnetperf.microsoft.com")) + { + Console.WriteLine(Convert.ToBase64String(cert.Export(X509ContentType.Pfx))); + } + } + } + } + } +} From 591712798921bf0453a80536c4c29e958ca6d631 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 7 Jan 2025 17:01:58 -0800 Subject: [PATCH 07/24] Move build step for CertHelper --- scripts/run_performance_job.py | 50 ++++++++++++++++++---------------- scripts/upload.py | 2 +- 2 files changed, 27 insertions(+), 25 deletions(-) diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 9214f447717..2d90bc48719 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -774,6 +774,32 @@ def run_performance_job(args: RunPerformanceJobArgs): getLogger().info("Copying global.json to payload directory") shutil.copy(os.path.join(args.performance_repo_dir, 'global.json'), os.path.join(performance_payload_dir, 'global.json')) + # Building CertHelper needs to happen here as we need it on every run. This also means that we will need to move the calculation + # of the parameters needed outside of the if block + + framework = os.environ["PERFLAB_Framework"] + os.environ["PERFLAB_TARGET_FRAMEWORKS"] = framework + if args.os_group == "windows": + runtime_id = f"win-{args.architecture}" + elif args.os_group == "osx": + runtime_id = f"osx-{args.architecture}" + else: + runtime_id = f"linux-{args.architecture}" + + dotnet_executable_path = os.path.join(ci_setup_arguments.install_dir, "dotnet") + + RunCommand([ + dotnet_executable_path, "publish", + "-c", "Release", + "-o", os.path.join(payload_dir, "certhelper"), + "-f", framework, + "-r", runtime_id, + "--self-contained", + os.path.join(args.performance_repo_dir, "src", "tools", "CertHelper", "CertHelper.csproj"), + f"/bl:{os.path.join(args.performance_repo_dir, 'artifacts', 'log', build_config, 'CertHelper.binlog')}", + "-p:DisableTransitiveFrameworkReferenceDownloads=true"], + verbose=True).run() + if args.is_scenario: set_environment_variable("DOTNET_ROOT", ci_setup_arguments.install_dir, save_to_pipeline=True) getLogger().info(f"Set DOTNET_ROOT to {ci_setup_arguments.install_dir}") @@ -782,17 +808,6 @@ def run_performance_job(args: RunPerformanceJobArgs): set_environment_variable("PATH", new_path, save_to_pipeline=True) getLogger().info(f"Set PATH to {new_path}") - framework = os.environ["PERFLAB_Framework"] - os.environ["PERFLAB_TARGET_FRAMEWORKS"] = framework - if args.os_group == "windows": - runtime_id = f"win-{args.architecture}" - elif args.os_group == "osx": - runtime_id = f"osx-{args.architecture}" - else: - runtime_id = f"linux-{args.architecture}" - - dotnet_executable_path = os.path.join(ci_setup_arguments.install_dir, "dotnet") - os.environ["MSBUILDDISABLENODEREUSE"] = "1" # without this, MSbuild will be kept alive # build Startup @@ -807,19 +822,6 @@ def run_performance_job(args: RunPerformanceJobArgs): f"/bl:{os.path.join(args.performance_repo_dir, 'artifacts', 'log', build_config, 'Startup.binlog')}", "-p:DisableTransitiveFrameworkReferenceDownloads=true"], verbose=True).run() - - # build CertHelper - RunCommand([ - dotnet_executable_path, "publish", - "-c", "Release", - "-o", os.path.join(payload_dir, "certhelper"), - "-f", framework, - "-r", runtime_id, - "--self-contained", - os.path.join(args.performance_repo_dir, "src", "tools", "CertHelper", "CertHelper.csproj"), - f"/bl:{os.path.join(args.performance_repo_dir, 'artifacts', 'log', build_config, 'CertHelper.binlog')}", - "-p:DisableTransitiveFrameworkReferenceDownloads=true"], - verbose=True).run() # build SizeOnDisk RunCommand([ diff --git a/scripts/upload.py b/scripts/upload.py index 3c64020f54a..440ea5f7238 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -39,7 +39,7 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag except ClientAuthenticationError as ex: credential = None getLogger().info("Unable to use managed identity. Falling back to certificate.") - cmd_line = list((os.path.join(str(helixpayload()), 'Microsoft.Dotnet.Performance.CertHelper.exe'))) + cmd_line = list((os.path.join(str(helixpayload()), 'certhelper', 'CertHelper.exe'))) cert_helper = RunCommand(cmd_line, None, False, 0) cert_helper.run() for cert in cert_helper.stdout.splitlines(): From 75e815d5fc20e987a3d40b703f17693087d722b8 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Wed, 8 Jan 2025 10:35:55 -0800 Subject: [PATCH 08/24] Address PR feedback --- scripts/upload.py | 6 ++---- src/tools/CertHelper/CertHelper.sln | 22 ++++++++++++++++++++++ src/tools/CertHelper/Program.cs | 7 ++----- 3 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 src/tools/CertHelper/CertHelper.sln diff --git a/scripts/upload.py b/scripts/upload.py index 440ea5f7238..cac529fbdba 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -6,12 +6,10 @@ from azure.identity import DefaultAzureCredential, ClientAssertionCredential, CertificateCredential from traceback import format_exc from glob import glob -from performance.common import retry_on_exception, iswin, islinux, RunCommand, helixpayload, base64_to_bytes +from performance.common import retry_on_exception, RunCommand, helixpayload, base64_to_bytes, extension from performance.constants import TENANT_ID, ARC_CLIENT_ID, CERT_CLIENT_ID import os import json -import ssl -from cryptography import x509 from logging import getLogger @@ -39,7 +37,7 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag except ClientAuthenticationError as ex: credential = None getLogger().info("Unable to use managed identity. Falling back to certificate.") - cmd_line = list((os.path.join(str(helixpayload()), 'certhelper', 'CertHelper.exe'))) + cmd_line = list((os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))) cert_helper = RunCommand(cmd_line, None, False, 0) cert_helper.run() for cert in cert_helper.stdout.splitlines(): diff --git a/src/tools/CertHelper/CertHelper.sln b/src/tools/CertHelper/CertHelper.sln new file mode 100644 index 00000000000..6528a7045b7 --- /dev/null +++ b/src/tools/CertHelper/CertHelper.sln @@ -0,0 +1,22 @@ + +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 17 +VisualStudioVersion = 17.12.35514.174 d17.12 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertHelper", "CertHelper.csproj", "{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection +EndGlobal diff --git a/src/tools/CertHelper/Program.cs b/src/tools/CertHelper/Program.cs index 1b0384f4615..305406349bd 100644 --- a/src/tools/CertHelper/Program.cs +++ b/src/tools/CertHelper/Program.cs @@ -9,12 +9,9 @@ static void Main(string[] args) { using(var store = new X509Store(StoreName.My, StoreLocation.CurrentUser, OpenFlags.ReadWrite)) { - foreach(var cert in store.Certificates) + foreach(var cert in store.Certificates.Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false)) { - if (cert.Subject.Contains("CN=dotnetperf.microsoft.com")) - { - Console.WriteLine(Convert.ToBase64String(cert.Export(X509ContentType.Pfx))); - } + Console.WriteLine(Convert.ToBase64String(cert.Export(X509ContentType.Pfx))); } } } From 363f723e0f19dfd871452d115abeaf9aa60d5c6e Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Wed, 8 Jan 2025 10:45:57 -0800 Subject: [PATCH 09/24] Remove islinux --- scripts/performance/common.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/scripts/performance/common.py b/scripts/performance/common.py index 5fcfba6e038..c03aeca8533 100644 --- a/scripts/performance/common.py +++ b/scripts/performance/common.py @@ -39,9 +39,6 @@ def get_machine_architecture(): def iswin(): return sys.platform == 'win32' -def islinux(): - return sys.platform == 'linux' - def extension(): 'gets platform specific extension' return '.exe' if iswin() else '' From 61f080207301aeb67daa6d187140205df3f3a811 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Wed, 8 Jan 2025 15:15:10 -0800 Subject: [PATCH 10/24] Add CertRotator to CertHelper --- src/tools/CertHelper/AssemblyInfo.cs | 12 ++ src/tools/CertHelper/CertHelper.csproj | 11 +- src/tools/CertHelper/CertHelper.sln | 6 + src/tools/CertHelper/Constants.cs | 14 ++ src/tools/CertHelper/IX509Store.cs | 35 ++++ src/tools/CertHelper/KeyVaultCert.cs | 106 +++++++++++ src/tools/CertHelper/LocalCert.cs | 44 +++++ src/tools/CertHelper/Program.cs | 40 ++++- .../CertHelperTests/CertRotatorTests.csproj | 30 ++++ .../CertHelperTests/KeyVaultCertTests.cs | 170 ++++++++++++++++++ src/tools/CertHelperTests/LocalCertTests.cs | 112 ++++++++++++ 11 files changed, 573 insertions(+), 7 deletions(-) create mode 100644 src/tools/CertHelper/AssemblyInfo.cs create mode 100644 src/tools/CertHelper/Constants.cs create mode 100644 src/tools/CertHelper/IX509Store.cs create mode 100644 src/tools/CertHelper/KeyVaultCert.cs create mode 100644 src/tools/CertHelper/LocalCert.cs create mode 100644 src/tools/CertHelperTests/CertRotatorTests.csproj create mode 100644 src/tools/CertHelperTests/KeyVaultCertTests.cs create mode 100644 src/tools/CertHelperTests/LocalCertTests.cs diff --git a/src/tools/CertHelper/AssemblyInfo.cs b/src/tools/CertHelper/AssemblyInfo.cs new file mode 100644 index 00000000000..edd1d789f59 --- /dev/null +++ b/src/tools/CertHelper/AssemblyInfo.cs @@ -0,0 +1,12 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Text; +using System.Threading.Tasks; + +[assembly: InternalsVisibleTo("CertHelperTests")] +namespace CertHelper; +internal class AssemblyInfo +{ +} diff --git a/src/tools/CertHelper/CertHelper.csproj b/src/tools/CertHelper/CertHelper.csproj index 2150e3797ba..cc55f43ceac 100644 --- a/src/tools/CertHelper/CertHelper.csproj +++ b/src/tools/CertHelper/CertHelper.csproj @@ -2,9 +2,18 @@ Exe - net8.0 + $(PERFLAB_TARGET_FRAMEWORKS) + + net8.0 enable enable + + + + + + + diff --git a/src/tools/CertHelper/CertHelper.sln b/src/tools/CertHelper/CertHelper.sln index 6528a7045b7..df485270020 100644 --- a/src/tools/CertHelper/CertHelper.sln +++ b/src/tools/CertHelper/CertHelper.sln @@ -5,6 +5,8 @@ VisualStudioVersion = 17.12.35514.174 d17.12 MinimumVisualStudioVersion = 10.0.40219.1 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertHelper", "CertHelper.csproj", "{165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "CertRotatorTests", "..\CertHelperTests\CertRotatorTests.csproj", "{AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -15,6 +17,10 @@ Global {165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Debug|Any CPU.Build.0 = Debug|Any CPU {165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.ActiveCfg = Release|Any CPU {165A37BD-2E9E-4D0A-8402-BB58C29A0BF4}.Release|Any CPU.Build.0 = Release|Any CPU + {AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AEA0F93B-EC9B-4438-991E-A80C0C82B3D1}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/tools/CertHelper/Constants.cs b/src/tools/CertHelper/Constants.cs new file mode 100644 index 00000000000..691006fab9d --- /dev/null +++ b/src/tools/CertHelper/Constants.cs @@ -0,0 +1,14 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace CertHelper; +public class Constants +{ + public static readonly string Cert1Name = "LabCert1"; + public static readonly string Cert2Name = "LabCert2"; + public static readonly Uri Cert1Id = new Uri("https://test.vault.azure.net/certificates/LabCert1/07a7d98bf4884e5c40e690e02b96b3b4"); + public static readonly Uri Cert2Id = new Uri("https://test.vault.azure.net/certificates/LabCert2/07a7d98bf4884e5c40e690e02b96b3b4"); +} diff --git a/src/tools/CertHelper/IX509Store.cs b/src/tools/CertHelper/IX509Store.cs new file mode 100644 index 00000000000..4e59441302b --- /dev/null +++ b/src/tools/CertHelper/IX509Store.cs @@ -0,0 +1,35 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading.Tasks; + +namespace CertHelper; +public interface IX509Store +{ + X509Certificate2Collection Certificates { get; } + string? Name { get; } + StoreLocation Location { get; } + X509Store GetX509Store(); +} + +public class TestableX509Store : IX509Store +{ + public X509Certificate2Collection Certificates { get => store.Certificates; } + + public string? Name => store.Name; + + public StoreLocation Location => store.Location; + + private X509Store store; + public TestableX509Store(OpenFlags flags = OpenFlags.ReadOnly) + { + store = new X509Store(StoreName.My, StoreLocation.LocalMachine, flags); + } + + public X509Store GetX509Store() + { + return store; + } +} diff --git a/src/tools/CertHelper/KeyVaultCert.cs b/src/tools/CertHelper/KeyVaultCert.cs new file mode 100644 index 00000000000..b2fd7819ff1 --- /dev/null +++ b/src/tools/CertHelper/KeyVaultCert.cs @@ -0,0 +1,106 @@ +using Azure; +using Azure.Core; +using Azure.Identity; +using Azure.Security.KeyVault.Certificates; +using Azure.Security.KeyVault.Secrets; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading.Tasks; + +namespace CertHelper; + +public class KeyVaultCert +{ + private readonly string KeyVaultUrl = "https://dotnetperfkeyvault.vault.azure.net/"; + private readonly string TenantId = "72f988bf-86f1-41af-91ab-2d7cd011db47"; + private readonly string ClientId = "8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc"; + + public X509Certificate2Collection KeyVaultCertificates { get; set; } + public ILocalCert LocalCerts { get; set; } + private TokenCredential Credential { get; set; } + private CertificateClient CertClient { get; set; } + private SecretClient SecretClient { get; set; } + + public KeyVaultCert(TokenCredential? cred = null, CertificateClient? certClient = null, SecretClient? secretClient = null, ILocalCert? localCerts = null) + { + LocalCerts = localCerts ?? new LocalCert(); + Credential = cred ?? GetCertifcateCredentialAsync(TenantId, ClientId, LocalCerts.Certificates).Result; + CertClient = certClient ?? new CertificateClient(new Uri(KeyVaultUrl), Credential); + SecretClient = secretClient ?? new SecretClient(new Uri(KeyVaultUrl), Credential); + KeyVaultCertificates = new X509Certificate2Collection(); + } + + public async Task GetKeyVaultCerts() + { + KeyVaultCertificates.Add(await FindCertificateInKeyVault(Constants.Cert1Name)); + KeyVaultCertificates.Add(await FindCertificateInKeyVault(Constants.Cert2Name)); + + if (KeyVaultCertificates.Count != 2 || KeyVaultCertificates.Where(c => c == null).Count() > 0) + { + throw new Exception("One or more certificates not found"); + } + } + + private async Task GetCertifcateCredentialAsync(string tenantId, string clientId, X509Certificate2Collection certCollection) + { + ClientCertificateCredential? ccc = null; + Exception? exception = null; + foreach (var cert in certCollection) + { + try + { + ccc = new ClientCertificateCredential(tenantId, clientId, cert); + await ccc.GetTokenAsync(new TokenRequestContext(new string[] { "https://vault.azure.net/.default" })); + break; + } + catch (Exception ex) + { + ccc = null; + exception = ex; + } + } + if(ccc == null) + { + throw new Exception("Both certificates failed to authenticate", exception); + } + return ccc; + } + + private async Task FindCertificateInKeyVault(string certName) + { + var keyVaultCert = await CertClient.GetCertificateAsync(certName); + if(keyVaultCert.Value == null) + { + throw new Exception("Certificate not found in Key Vault"); + } + var secret = await SecretClient.GetSecretAsync(keyVaultCert.Value.Name, keyVaultCert.Value.SecretId.Segments.Last()); + if(secret.Value == null) + { + throw new Exception("Certificate secret not found in Key Vault"); + } + var certBytes = Convert.FromBase64String(secret.Value.Value); + var cert = new X509Certificate2(certBytes, "", X509KeyStorageFlags.Exportable); + return cert; + } + + public bool ShouldRotateCerts() + { + var keyVaultThumbprints = new HashSet(); + foreach (var cert in KeyVaultCertificates) + { + keyVaultThumbprints.Add(cert.Thumbprint); + } + foreach(var cert in LocalCerts.Certificates) + { + if (!keyVaultThumbprints.Contains(cert.Thumbprint)) + { + return true; + } + } + return false; + } +} diff --git a/src/tools/CertHelper/LocalCert.cs b/src/tools/CertHelper/LocalCert.cs new file mode 100644 index 00000000000..3459e7509df --- /dev/null +++ b/src/tools/CertHelper/LocalCert.cs @@ -0,0 +1,44 @@ +using Azure.Security.KeyVault.Certificates; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.CompilerServices; +using System.Security.Cryptography.X509Certificates; +using System.Text; +using System.Threading.Tasks; + +namespace CertHelper; + +public class LocalCert : ILocalCert +{ + public X509Certificate2Collection Certificates { get; set; } + internal IX509Store LocalMachineCerts { get; set; } + + public LocalCert(IX509Store? store = null) + { + LocalMachineCerts = store ?? new TestableX509Store(); + Certificates = new X509Certificate2Collection(); + GetLocalCerts(); + } + + private void GetLocalCerts() + { + foreach (var cert in LocalMachineCerts.Certificates.Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false)) + { + if (cert.Subject == "CN=dotnetperf.microsoft.com") + { + Certificates.Add(cert); + } + } + + if (Certificates.Count < 2 || Certificates.Where(c => c == null).Count() > 0) + { + throw new Exception("One or more certificates not found"); + } + } +} + +public interface ILocalCert +{ + X509Certificate2Collection Certificates { get; set; } +} diff --git a/src/tools/CertHelper/Program.cs b/src/tools/CertHelper/Program.cs index 305406349bd..107300dbae4 100644 --- a/src/tools/CertHelper/Program.cs +++ b/src/tools/CertHelper/Program.cs @@ -1,19 +1,47 @@ using System.Security.Cryptography.X509Certificates; using System.Text; -namespace CertHelper +namespace CertHelper; + +internal class Program { - internal class Program + static async Task Main(string[] args) { - static void Main(string[] args) + var kvc = new KeyVaultCert(); + await kvc.GetKeyVaultCerts(); + var updated = false; + if (kvc.ShouldRotateCerts()) { - using(var store = new X509Store(StoreName.My, StoreLocation.CurrentUser, OpenFlags.ReadWrite)) + using (var localMachineCerts = new X509Store(StoreName.My, StoreLocation.LocalMachine)) { - foreach(var cert in store.Certificates.Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false)) + try + { + localMachineCerts.Open(OpenFlags.ReadWrite); + } + catch (System.Security.Cryptography.CryptographicException ex) when (ex.Message.Contains("Unix")) + { + var localMachineCertsCurrentUser = new X509Store(StoreName.My, StoreLocation.CurrentUser); + localMachineCertsCurrentUser.Open(OpenFlags.ReadWrite); + localMachineCertsCurrentUser.RemoveRange(kvc.LocalCerts.Certificates); + localMachineCertsCurrentUser.AddRange(kvc.KeyVaultCertificates); + localMachineCertsCurrentUser.Close(); + updated = true; + } + if(!updated) { - Console.WriteLine(Convert.ToBase64String(cert.Export(X509ContentType.Pfx))); + localMachineCerts.RemoveRange(kvc.LocalCerts.Certificates); + localMachineCerts.AddRange(kvc.KeyVaultCertificates); } } } + + using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser, OpenFlags.ReadWrite)) + { + foreach(var cert in store.Certificates.Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false)) + { + Console.WriteLine(Convert.ToBase64String(cert.Export(X509ContentType.Pfx))); + } + } + return 0; } } diff --git a/src/tools/CertHelperTests/CertRotatorTests.csproj b/src/tools/CertHelperTests/CertRotatorTests.csproj new file mode 100644 index 00000000000..365b1673c9b --- /dev/null +++ b/src/tools/CertHelperTests/CertRotatorTests.csproj @@ -0,0 +1,30 @@ + + + + net8.0 + enable + enable + + false + true + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + diff --git a/src/tools/CertHelperTests/KeyVaultCertTests.cs b/src/tools/CertHelperTests/KeyVaultCertTests.cs new file mode 100644 index 00000000000..9f09d473e0d --- /dev/null +++ b/src/tools/CertHelperTests/KeyVaultCertTests.cs @@ -0,0 +1,170 @@ +using System; +using System.Linq; +using System.Reflection; +using System.Security.Cryptography; +using System.Security.Cryptography.X509Certificates; +using System.Threading.Tasks; +using Azure; +using Azure.Core; +using Azure.Identity; +using Azure.Security.KeyVault.Certificates; +using Azure.Security.KeyVault.Secrets; +using Moq; +using Xunit; +using Xunit.Sdk; + +namespace CertHelper.Tests; + +public class KeyVaultCertTests +{ + [Fact] + public async Task GetKeyVaultCerts_ShouldAddCertificatesToCollection() + { + // Arrange + Mock mockTokenCred; + Mock mockCertClient; + Mock mockSecretClient; + Mock mockLocalCert; + CertStoreSetup(out mockTokenCred, out mockCertClient, out mockSecretClient, out mockLocalCert, false); + + var keyVaultCert = new KeyVaultCert(mockTokenCred.Object, mockCertClient.Object, mockSecretClient.Object, mockLocalCert.Object); + + // Act + await keyVaultCert.GetKeyVaultCerts(); + + // Assert + Assert.Equal(2, keyVaultCert.KeyVaultCertificates.Count); + } + + private static void CertStoreSetup(out Mock mockTokenCred, out Mock mockCertClient, out Mock mockSecretClient, out Mock mockLocalCert, bool missingKeyVaultCerts = false, bool localAndKeyVaultDifferent = false) + { + mockTokenCred = new Mock(); + mockTokenCred.Setup(tc => tc.GetTokenAsync(It.IsAny(), default)).ReturnsAsync(new AccessToken("token", DateTimeOffset.Now)); + + mockCertClient = new Mock(new Uri("https://dotnetperfkeyvault.vault.azure.net/"), mockTokenCred.Object); + mockSecretClient = new Mock(new Uri("https://dotnetperfkeyvault.vault.azure.net/"), mockTokenCred.Object); + KeyVaultCertificateWithPolicy? mockCert1, mockCert2; + X509Certificate2 cert1, cert2; + MakeCerts(out mockCert1, out mockCert2, out cert1, out cert2, localAndKeyVaultDifferent); + + var certCollection = new X509Certificate2Collection { cert1, cert2 }; + +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert1Name, default)).ReturnsAsync(Response.FromValue(mockCert1, null)); +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + if (missingKeyVaultCerts) + { +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert2Name, default)).ReturnsAsync(Response.FromValue(null, null)); +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + } + else + { +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert2Name, default)).ReturnsAsync(Response.FromValue(mockCert2, null)); +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + } + + //var secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(certCollection[0].Export(X509ContentType.Cert))); + //var secret2 = new KeyVaultSecret(Constants.Cert2Name, Convert.ToBase64String(certCollection[1].Export(X509ContentType.Cert))); +#pragma warning disable CS8602 // Dereference of a possibly null reference. + var secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(mockCert1.Cer)); +#pragma warning restore CS8602 // Dereference of a possibly null reference. +#pragma warning disable CS8602 // Dereference of a possibly null reference. + var secret2 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(mockCert2.Cer)); +#pragma warning restore CS8602 // Dereference of a possibly null reference. + +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + mockSecretClient.Setup(s => s.GetSecretAsync(Constants.Cert1Name, mockCert1.SecretId.Segments.Last(), default)).ReturnsAsync(Response.FromValue(secret1, null)); +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. +#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. + mockSecretClient.Setup(s => s.GetSecretAsync(Constants.Cert2Name, mockCert2.SecretId.Segments.Last(), default)).ReturnsAsync(Response.FromValue(secret2, null)); +#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + + mockLocalCert = new Mock(); + mockLocalCert.Setup(lc => lc.Certificates).Returns(certCollection); + } + + private static void MakeCerts(out KeyVaultCertificateWithPolicy? mockCert1, out KeyVaultCertificateWithPolicy? mockCert2, out X509Certificate2 cert1, out X509Certificate2 cert2, bool localAndKeyVaultDifferent = false) + { + var ecdsa1 = ECDsa.Create(); // generate asymmetric key pair + var req1 = new CertificateRequest("cn=perflabtest", ecdsa1, HashAlgorithmName.SHA256); + var tmpCert1 = req1.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + var ecdsa2 = ECDsa.Create(); // generate asymmetric key pair + var req2 = new CertificateRequest("cn=perflabtest", ecdsa2, HashAlgorithmName.SHA256); + var tmpCert2 = req2.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + cert1 = tmpCert1; + cert2 = tmpCert2; + + if(localAndKeyVaultDifferent) + { + var ecdsa = ECDsa.Create(); // generate asymmetric key pair + var req = new CertificateRequest("cn=perflabtest", ecdsa, HashAlgorithmName.SHA256); + tmpCert1 = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + } + + mockCert1 = CertificateModelFactory.KeyVaultCertificateWithPolicy(CertificateModelFactory.CertificateProperties(Constants.Cert1Id, Constants.Cert1Name, x509thumbprint: Convert.FromHexString(tmpCert1.Thumbprint)), + Constants.Cert1Id, Constants.Cert1Id, tmpCert1.GetRawCertData()); + mockCert2 = CertificateModelFactory.KeyVaultCertificateWithPolicy(CertificateModelFactory.CertificateProperties(Constants.Cert2Id, Constants.Cert2Name, x509thumbprint: Convert.FromHexString(tmpCert2.Thumbprint)), + Constants.Cert2Id, Constants.Cert2Id, tmpCert2.GetRawCertData()); + } + + [Fact] + public async Task GetKeyVaultCerts_ShouldThrowException_WhenCertificatesNotFound() + { + // Arrange + Mock mockTokenCred; + Mock mockCertClient; + Mock mockSecretClient; + Mock mockLocalCert; + CertStoreSetup(out mockTokenCred, out mockCertClient, out mockSecretClient, out mockLocalCert, missingKeyVaultCerts: true); + + var keyVaultCert = new KeyVaultCert(mockTokenCred.Object, mockCertClient.Object, mockSecretClient.Object, mockLocalCert.Object); + + // Act & Assert + await Assert.ThrowsAsync(() => keyVaultCert.GetKeyVaultCerts()); + } + + [Fact] + public async Task ShouldRotateCerts_ShouldReturnTrue_WhenThumbprintsDoNotMatch() + { + // Arrange + Mock mockTokenCred; + Mock mockCertClient; + Mock mockSecretClient; + Mock mockLocalCert; + CertStoreSetup(out mockTokenCred, out mockCertClient, out mockSecretClient, out mockLocalCert, localAndKeyVaultDifferent: true); + + var keyVaultCert = new KeyVaultCert(mockTokenCred.Object, mockCertClient.Object, mockSecretClient.Object, mockLocalCert.Object); + + // Act + await keyVaultCert.GetKeyVaultCerts(); + + var result = keyVaultCert.ShouldRotateCerts(); + + // Assert + Assert.True(result); + } + + [Fact] + public async Task ShouldRotateCerts_ShouldReturnFalse_WhenThumbprintsMatch() + { + // Arrange + Mock mockTokenCred; + Mock mockCertClient; + Mock mockSecretClient; + Mock mockLocalCert; + CertStoreSetup(out mockTokenCred, out mockCertClient, out mockSecretClient, out mockLocalCert); + + var keyVaultCert = new KeyVaultCert(mockTokenCred.Object, mockCertClient.Object, mockSecretClient.Object, mockLocalCert.Object); + + // Act + await keyVaultCert.GetKeyVaultCerts(); + + var result = keyVaultCert.ShouldRotateCerts(); + + // Assert + Assert.False(result); + } +} + diff --git a/src/tools/CertHelperTests/LocalCertTests.cs b/src/tools/CertHelperTests/LocalCertTests.cs new file mode 100644 index 00000000000..fcc49ad596e --- /dev/null +++ b/src/tools/CertHelperTests/LocalCertTests.cs @@ -0,0 +1,112 @@ +using System.Security.Cryptography.X509Certificates; +using Xunit; +using Moq; +using Microsoft.VisualBasic; +using System.Security.Cryptography; + +namespace CertHelper.Tests; + +public class LocalCertTests +{ + [Fact] + public void GetLocalCerts_ShouldAddCertificatesToCollection() + { + // Arrange + var mockStore = new Mock(); + var ecdsa1 = ECDsa.Create(); // generate asymmetric key pair + var req1 = new CertificateRequest("CN=dotnetperf.microsoft.com", ecdsa1, HashAlgorithmName.SHA256); + var cert1 = req1.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + var ecdsa2 = ECDsa.Create(); // generate asymmetric key pair + var req2 = new CertificateRequest("CN=dotnetperf.microsoft.com", ecdsa2, HashAlgorithmName.SHA256); + var cert2 = req2.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + + var mockCert1 = cert1; + var mockCert2 = cert2; + + var certCollection = new X509Certificate2Collection { mockCert1, mockCert2 }; + mockStore.Setup(s => s.Certificates).Returns(certCollection); + + + + // Act + var localCert = new LocalCert(mockStore.Object); + + // Assert + Assert.Equal(2, localCert.Certificates.Count); + Assert.Contains(mockCert1, localCert.Certificates); + Assert.Contains(mockCert2, localCert.Certificates); + } + + [Fact] + public void GetLocalCerts_ShouldAddCertificatesToCollection_WhenSubjectMatches() + { + // Arrange + var mockStore = new Mock(); + var ecdsa1 = ECDsa.Create(); // generate asymmetric key pair + var req1 = new CertificateRequest("CN=dotnetperf.microsoft.com", ecdsa1, HashAlgorithmName.SHA256); + var cert1 = req1.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + var ecdsa2 = ECDsa.Create(); // generate asymmetric key pair + var req2 = new CertificateRequest("CN=dotnetperf.microsoft.com", ecdsa2, HashAlgorithmName.SHA256); + var cert2 = req2.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + + var mockCert1 = cert1; + var mockCert2 = cert2; + + var certCollection = new X509Certificate2Collection { mockCert1, mockCert2 }; + mockStore.Setup(s => s.Certificates).Returns(certCollection); + + // Act + var localCert = new LocalCert(mockStore.Object); + + // Assert + Assert.Equal(2, localCert.Certificates.Count); + Assert.Contains(mockCert1, localCert.Certificates); + Assert.Contains(mockCert2, localCert.Certificates); + } + + [Fact] + public void GetLocalCerts_ShouldThrowException_WhenOneCertificateFound() + { + // Arrange + var mockStore = new Mock(); + var ecdsa1 = ECDsa.Create(); // generate asymmetric key pair + var req1 = new CertificateRequest("CN=dotnetperf.microsoft.com", ecdsa1, HashAlgorithmName.SHA256); + var cert1 = req1.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + var certCollection = new X509Certificate2Collection { cert1 }; + mockStore.Setup(s => s.Certificates).Returns(certCollection); + + // Act & Assert + Assert.Throws(() => new LocalCert(mockStore.Object)); + } + + [Fact] + public void GetLocalCerts_ShouldThrowException_WhenCertificatesHaveWrongSubject() + { + // Arrange + var mockStore = new Mock(); + var ecdsa1 = ECDsa.Create(); // generate asymmetric key pair + var req1 = new CertificateRequest("CN=dotnetperf.microsoft.co", ecdsa1, HashAlgorithmName.SHA256); + var cert1 = req1.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + var ecdsa2 = ECDsa.Create(); // generate asymmetric key pair + var req2 = new CertificateRequest("CN=dotnetperf.microsoft.co", ecdsa1, HashAlgorithmName.SHA256); + var cert2 = req1.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + + var certCollection = new X509Certificate2Collection { cert1, cert2 }; + mockStore.Setup(s => s.Certificates).Returns(certCollection); + + // Act & Assert + Assert.Throws(() => new LocalCert(mockStore.Object)); + } + + [Fact] + public void GetLocalCerts_ShouldThrowException_WhenCertificatesNotFound() + { + // Arrange + var mockStore = new Mock(); + var certCollection = new X509Certificate2Collection(); + mockStore.Setup(s => s.Certificates).Returns(certCollection); + + // Act & Assert + Assert.Throws(() => new LocalCert(mockStore.Object)); + } +} From ee2dd677cd88442a7a3efa3df03c22e1c1443e08 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Wed, 8 Jan 2025 16:44:32 -0800 Subject: [PATCH 11/24] WIP --- src/tools/CertHelper/CertHelper.csproj | 2 +- src/tools/CertHelper/KeyVaultCert.cs | 4 ++++ .../CertHelperTests/CertRotatorTests.csproj | 2 +- src/tools/CertHelperTests/KeyVaultCertTests.cs | 18 +++++++++++------- 4 files changed, 17 insertions(+), 9 deletions(-) diff --git a/src/tools/CertHelper/CertHelper.csproj b/src/tools/CertHelper/CertHelper.csproj index cc55f43ceac..185bb4c2ef8 100644 --- a/src/tools/CertHelper/CertHelper.csproj +++ b/src/tools/CertHelper/CertHelper.csproj @@ -4,7 +4,7 @@ Exe $(PERFLAB_TARGET_FRAMEWORKS) - net8.0 + net9.0 enable enable diff --git a/src/tools/CertHelper/KeyVaultCert.cs b/src/tools/CertHelper/KeyVaultCert.cs index b2fd7819ff1..d72a652b8c7 100644 --- a/src/tools/CertHelper/KeyVaultCert.cs +++ b/src/tools/CertHelper/KeyVaultCert.cs @@ -83,7 +83,11 @@ private async Task FindCertificateInKeyVault(string certName) throw new Exception("Certificate secret not found in Key Vault"); } var certBytes = Convert.FromBase64String(secret.Value.Value); +#if NET9_0_OR_GREATER + var cert = X509CertificateLoader.LoadPkcs12(certBytes, "", X509KeyStorageFlags.Exportable); +#else var cert = new X509Certificate2(certBytes, "", X509KeyStorageFlags.Exportable); +#endif return cert; } diff --git a/src/tools/CertHelperTests/CertRotatorTests.csproj b/src/tools/CertHelperTests/CertRotatorTests.csproj index 365b1673c9b..3acaadb46a9 100644 --- a/src/tools/CertHelperTests/CertRotatorTests.csproj +++ b/src/tools/CertHelperTests/CertRotatorTests.csproj @@ -1,7 +1,7 @@  - net8.0 + net9.0 enable enable diff --git a/src/tools/CertHelperTests/KeyVaultCertTests.cs b/src/tools/CertHelperTests/KeyVaultCertTests.cs index 9f09d473e0d..b56e59a304a 100644 --- a/src/tools/CertHelperTests/KeyVaultCertTests.cs +++ b/src/tools/CertHelperTests/KeyVaultCertTests.cs @@ -68,17 +68,21 @@ private static void CertStoreSetup(out Mock mockTokenCred, out //var secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(certCollection[0].Export(X509ContentType.Cert))); //var secret2 = new KeyVaultSecret(Constants.Cert2Name, Convert.ToBase64String(certCollection[1].Export(X509ContentType.Cert))); #pragma warning disable CS8602 // Dereference of a possibly null reference. - var secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(mockCert1.Cer)); + var secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(cert1.Export(X509ContentType.Pfx))); #pragma warning restore CS8602 // Dereference of a possibly null reference. #pragma warning disable CS8602 // Dereference of a possibly null reference. - var secret2 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(mockCert2.Cer)); + var secret2 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(cert1.Export(X509ContentType.Pfx))); #pragma warning restore CS8602 // Dereference of a possibly null reference. #pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. +#pragma warning disable CS8602 // Dereference of a possibly null reference. mockSecretClient.Setup(s => s.GetSecretAsync(Constants.Cert1Name, mockCert1.SecretId.Segments.Last(), default)).ReturnsAsync(Response.FromValue(secret1, null)); +#pragma warning restore CS8602 // Dereference of a possibly null reference. #pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. #pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. +#pragma warning disable CS8602 // Dereference of a possibly null reference. mockSecretClient.Setup(s => s.GetSecretAsync(Constants.Cert2Name, mockCert2.SecretId.Segments.Last(), default)).ReturnsAsync(Response.FromValue(secret2, null)); +#pragma warning restore CS8602 // Dereference of a possibly null reference. #pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. mockLocalCert = new Mock(); @@ -87,11 +91,11 @@ private static void CertStoreSetup(out Mock mockTokenCred, out private static void MakeCerts(out KeyVaultCertificateWithPolicy? mockCert1, out KeyVaultCertificateWithPolicy? mockCert2, out X509Certificate2 cert1, out X509Certificate2 cert2, bool localAndKeyVaultDifferent = false) { - var ecdsa1 = ECDsa.Create(); // generate asymmetric key pair - var req1 = new CertificateRequest("cn=perflabtest", ecdsa1, HashAlgorithmName.SHA256); + using var rsa1 = RSA.Create(); // generate asymmetric key pair + var req1 = new CertificateRequest("cn=perflabtest", rsa1, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); var tmpCert1 = req1.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); - var ecdsa2 = ECDsa.Create(); // generate asymmetric key pair - var req2 = new CertificateRequest("cn=perflabtest", ecdsa2, HashAlgorithmName.SHA256); + using var rsa2 = RSA.Create(); // generate asymmetric key pair + var req2 = new CertificateRequest("cn=perflabtest", rsa2, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); var tmpCert2 = req2.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); cert1 = tmpCert1; cert2 = tmpCert2; @@ -100,7 +104,7 @@ private static void MakeCerts(out KeyVaultCertificateWithPolicy? mockCert1, out { var ecdsa = ECDsa.Create(); // generate asymmetric key pair var req = new CertificateRequest("cn=perflabtest", ecdsa, HashAlgorithmName.SHA256); - tmpCert1 = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); + tmpCert1 = X509CertificateLoader.LoadPkcs12(req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)).Export(X509ContentType.Pkcs12), ""); } mockCert1 = CertificateModelFactory.KeyVaultCertificateWithPolicy(CertificateModelFactory.CertificateProperties(Constants.Cert1Id, Constants.Cert1Name, x509thumbprint: Convert.FromHexString(tmpCert1.Thumbprint)), From 0e9d28505249e3738055a1d710d2efb6af8051f7 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Jan 2025 10:49:28 -0800 Subject: [PATCH 12/24] Fix improper list() usage --- scripts/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upload.py b/scripts/upload.py index cac529fbdba..ce5c2293b2a 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -37,7 +37,7 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag except ClientAuthenticationError as ex: credential = None getLogger().info("Unable to use managed identity. Falling back to certificate.") - cmd_line = list((os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))) + cmd_line = list([(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))]) cert_helper = RunCommand(cmd_line, None, False, 0) cert_helper.run() for cert in cert_helper.stdout.splitlines(): From f610ce8efd520be272918bb66a2fc2f7523f6f47 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Jan 2025 10:53:20 -0800 Subject: [PATCH 13/24] Yaml testing changes --- azure-pipelines.yml | 310 ++++++++++++++++++++++---------------------- scripts/upload.py | 2 +- 2 files changed, 156 insertions(+), 156 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6a6593bb439..3364ff40c5c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -459,135 +459,135 @@ jobs: # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job - ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: - # SDK scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - - win-x86 - #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available - isPublic: false - jobParameters: - kind: sdk_scenarios - projectFile: sdk_scenarios.proj - channels: - - main - - 8.0 + # # SDK scenario benchmarks + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/scenarios.yml + # buildMachines: + # - win-x64 + # - win-x86 + # #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available + # isPublic: false + # jobParameters: + # kind: sdk_scenarios + # projectFile: sdk_scenarios.proj + # channels: + # - main + # - 8.0 - # Blazor 3.2 scenario benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/scenarios.yml - buildMachines: - - win-x64 - isPublic: false - jobParameters: - kind: blazor_scenarios - projectFile: blazor_scenarios.proj - channels: - - main - - 8.0 + # # Blazor 3.2 scenario benchmarks + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/scenarios.yml + # buildMachines: + # - win-x64 + # isPublic: false + # jobParameters: + # kind: blazor_scenarios + # projectFile: blazor_scenarios.proj + # channels: + # - main + # - 8.0 - # F# benchmarks - - ${{ if false }}: # skipping, no useful benchmarks there currently - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: fsharp - csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - runCategories: 'fsharp' - channels: - - main - - 8.0 + # # F# benchmarks + # - ${{ if false }}: # skipping, no useful benchmarks there currently + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/benchmark_jobs.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # - win-arm64 + # - ubuntu-arm64-ampere + # isPublic: false + # jobParameters: + # kind: fsharp + # csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + # runCategories: 'fsharp' + # channels: + # - main + # - 8.0 - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: fsharpmicro - csproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj - runCategories: 'FSharpMicro' - channels: - - main - - 8.0 + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/benchmark_jobs.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # - win-arm64 + # - ubuntu-arm64-ampere + # isPublic: false + # jobParameters: + # kind: fsharpmicro + # csproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + # runCategories: 'FSharpMicro' + # channels: + # - main + # - 8.0 - # bepuphysics benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: bepuphysics - csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - runCategories: 'BepuPhysics' - channels: - - main - - 8.0 + # # bepuphysics benchmarks + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/benchmark_jobs.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # - win-arm64 + # - ubuntu-arm64-ampere + # isPublic: false + # jobParameters: + # kind: bepuphysics + # csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + # runCategories: 'BepuPhysics' + # channels: + # - main + # - 8.0 - # ImageSharp benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: imagesharp - csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - runCategories: 'ImageSharp' - channels: - - main - - 8.0 + # # ImageSharp benchmarks + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/benchmark_jobs.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # - win-arm64 + # - ubuntu-arm64-ampere + # isPublic: false + # jobParameters: + # kind: imagesharp + # csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + # runCategories: 'ImageSharp' + # channels: + # - main + # - 8.0 - # Akade.IndexedSet benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: akadeindexedset - csproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj - runCategories: 'AkadeIndexedSet' - channels: - - main + # # Akade.IndexedSet benchmarks + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/benchmark_jobs.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # - win-arm64 + # - ubuntu-arm64-ampere + # isPublic: false + # jobParameters: + # kind: akadeindexedset + # csproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + # runCategories: 'AkadeIndexedSet' + # channels: + # - main # ML.NET benchmarks - template: /eng/performance/build_machine_matrix.yml parameters: jobTemplate: /eng/performance/benchmark_jobs.yml buildMachines: - - win-x64 - - ubuntu-x64 + # - win-x64 + # - ubuntu-x64 - win-arm64 - win-arm64-ampere - - ubuntu-arm64-ampere + # - ubuntu-arm64-ampere isPublic: false jobParameters: kind: mlnet @@ -602,29 +602,29 @@ jobs: - DOTNET_GCHeapCount=4 - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - # Roslyn benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - win-arm64-ampere - - ubuntu-arm64-ampere - isPublic: false - jobParameters: - kind: roslyn - csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - runCategories: 'roslyn' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 8.0 - affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - runEnvVars: - - DOTNET_GCgen0size=410000 # ~4MB - - DOTNET_GCHeapCount=4 - - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + # # Roslyn benchmarks + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/benchmark_jobs.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # - win-arm64 + # - win-arm64-ampere + # - ubuntu-arm64-ampere + # isPublic: false + # jobParameters: + # kind: roslyn + # csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + # runCategories: 'roslyn' + # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + # - main + # - 8.0 + # affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + # runEnvVars: + # - DOTNET_GCgen0size=410000 # ~4MB + # - DOTNET_GCHeapCount=4 + # - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB # ILLink benchmarks # disabled because of: https://github.com/dotnet/performance/issues/3569 @@ -644,23 +644,23 @@ jobs: # - main # - 8.0 - # Powershell benchmarks - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - - win-arm64 - - ubuntu-arm64 - isPublic: false - jobParameters: - kind: powershell - csproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj - runCategories: 'Public Internal' - channels: - - main - - 8.0 + # # Powershell benchmarks + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/benchmark_jobs.yml + # buildMachines: + # # - win-x64 + # # - ubuntu-x64 + # - win-arm64 + # # - ubuntu-arm64 + # isPublic: false + # jobParameters: + # kind: powershell + # csproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + # runCategories: 'Public Internal' + # channels: + # - main + # - 8.0 # Secret Sync - job: Synchronize diff --git a/scripts/upload.py b/scripts/upload.py index ce5c2293b2a..ea2b00f76de 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -37,7 +37,7 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag except ClientAuthenticationError as ex: credential = None getLogger().info("Unable to use managed identity. Falling back to certificate.") - cmd_line = list([(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))]) + cmd_line = [(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))] cert_helper = RunCommand(cmd_line, None, False, 0) cert_helper.run() for cert in cert_helper.stdout.splitlines(): From 2654a1cb41396dee5fddd9f90ab024435dc36dc0 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 10 Jan 2025 11:34:26 -0800 Subject: [PATCH 14/24] Make RunCommand verbose --- scripts/upload.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/upload.py b/scripts/upload.py index ea2b00f76de..6d6e3bbcf95 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -38,7 +38,7 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag credential = None getLogger().info("Unable to use managed identity. Falling back to certificate.") cmd_line = [(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))] - cert_helper = RunCommand(cmd_line, None, False, 0) + cert_helper = RunCommand(cmd_line, None, True, 0) cert_helper.run() for cert in cert_helper.stdout.splitlines(): credential = CertificateCredential(TENANT_ID, CERT_CLIENT_ID, certificate_data=base64_to_bytes(cert)) From 3d77d4d9e87d4930e62b804e93a6e91051cc04cf Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 14 Jan 2025 09:25:46 -0800 Subject: [PATCH 15/24] Testing --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3364ff40c5c..ea3a30918d0 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -586,7 +586,7 @@ jobs: # - win-x64 # - ubuntu-x64 - win-arm64 - - win-arm64-ampere + # - win-arm64-ampere # - ubuntu-arm64-ampere isPublic: false jobParameters: From d545d7d8c408228761119ceafa9ae6b85103d994 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Tue, 14 Jan 2025 14:02:45 -0800 Subject: [PATCH 16/24] Add feature to RunCommand to not echo stdout --- scripts/performance/common.py | 10 +++++++++- scripts/upload.py | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/scripts/performance/common.py b/scripts/performance/common.py index c03aeca8533..ed125e5ab7e 100644 --- a/scripts/performance/common.py +++ b/scripts/performance/common.py @@ -239,6 +239,7 @@ def __init__( cmdline: List[str], success_exit_codes: Optional[List[int]] = None, verbose: bool = False, + echo: bool = True, retry: int = 0): if cmdline is None: raise TypeError('Unspecified command line to be executed.') @@ -248,6 +249,7 @@ def __init__( self.__cmdline = cmdline self.__verbose = verbose self.__retry = retry + self.__echo = echo if success_exit_codes is None: self.__success_exit_codes = [0] @@ -267,6 +269,11 @@ def success_exit_codes(self) -> List[int]: ''' return self.__success_exit_codes + @property + def echo(self) -> bool: + '''Enables/Disables echoing of STDOUT''' + return self.__echo + @property def verbose(self) -> bool: '''Enables/Disables verbosity.''' @@ -302,7 +309,8 @@ def __runinternal(self, working_directory: Optional[str] = None) -> Tuple[int, s line = raw_line.decode('utf-8', errors='backslashreplace') self.__stdout.write(line) line = line.rstrip() - getLogger().info(line) + if self.echo: + getLogger().info(line) proc.wait() return (proc.returncode, quoted_cmdline) diff --git a/scripts/upload.py b/scripts/upload.py index 6d6e3bbcf95..36bd40f57eb 100644 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -38,7 +38,7 @@ def upload(globpath: str, container: str, queue: str, sas_token_env: str, storag credential = None getLogger().info("Unable to use managed identity. Falling back to certificate.") cmd_line = [(os.path.join(str(helixpayload()), 'certhelper', "CertHelper%s" % extension()))] - cert_helper = RunCommand(cmd_line, None, True, 0) + cert_helper = RunCommand(cmd_line, None, True, False, 0) cert_helper.run() for cert in cert_helper.stdout.splitlines(): credential = CertificateCredential(TENANT_ID, CERT_CLIENT_ID, certificate_data=base64_to_bytes(cert)) From b846c3e7e547c963822e9b53af978fe385387b69 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 16 Jan 2025 15:47:45 -0800 Subject: [PATCH 17/24] Remove pragma and fixup tests --- src/tools/CertHelper/Constants.cs | 2 +- src/tools/CertHelper/IX509Store.cs | 2 +- src/tools/CertHelper/Program.cs | 29 ++++----- .../CertHelperTests/KeyVaultCertTests.cs | 60 ++++++++----------- 4 files changed, 39 insertions(+), 54 deletions(-) diff --git a/src/tools/CertHelper/Constants.cs b/src/tools/CertHelper/Constants.cs index 691006fab9d..3ddad191905 100644 --- a/src/tools/CertHelper/Constants.cs +++ b/src/tools/CertHelper/Constants.cs @@ -10,5 +10,5 @@ public class Constants public static readonly string Cert1Name = "LabCert1"; public static readonly string Cert2Name = "LabCert2"; public static readonly Uri Cert1Id = new Uri("https://test.vault.azure.net/certificates/LabCert1/07a7d98bf4884e5c40e690e02b96b3b4"); - public static readonly Uri Cert2Id = new Uri("https://test.vault.azure.net/certificates/LabCert2/07a7d98bf4884e5c40e690e02b96b3b4"); + public static readonly Uri Cert2Id = new Uri("https://test.vault.azure.net/certificates/LabCert2/07a7d98bf4884e5c41e690e02b96b3b4"); } diff --git a/src/tools/CertHelper/IX509Store.cs b/src/tools/CertHelper/IX509Store.cs index 4e59441302b..2fd187d86dc 100644 --- a/src/tools/CertHelper/IX509Store.cs +++ b/src/tools/CertHelper/IX509Store.cs @@ -25,7 +25,7 @@ public class TestableX509Store : IX509Store private X509Store store; public TestableX509Store(OpenFlags flags = OpenFlags.ReadOnly) { - store = new X509Store(StoreName.My, StoreLocation.LocalMachine, flags); + store = new X509Store(StoreName.My, StoreLocation.CurrentUser, flags); } public X509Store GetX509Store() diff --git a/src/tools/CertHelper/Program.cs b/src/tools/CertHelper/Program.cs index 107300dbae4..8ff23675bc1 100644 --- a/src/tools/CertHelper/Program.cs +++ b/src/tools/CertHelper/Program.cs @@ -7,33 +7,26 @@ internal class Program { static async Task Main(string[] args) { - var kvc = new KeyVaultCert(); - await kvc.GetKeyVaultCerts(); - var updated = false; - if (kvc.ShouldRotateCerts()) + try { - using (var localMachineCerts = new X509Store(StoreName.My, StoreLocation.LocalMachine)) + var kvc = new KeyVaultCert(); + await kvc.GetKeyVaultCerts(); + if (kvc.ShouldRotateCerts()) { - try + using (var localMachineCerts = new X509Store(StoreName.My, StoreLocation.CurrentUser)) { localMachineCerts.Open(OpenFlags.ReadWrite); - } - catch (System.Security.Cryptography.CryptographicException ex) when (ex.Message.Contains("Unix")) - { - var localMachineCertsCurrentUser = new X509Store(StoreName.My, StoreLocation.CurrentUser); - localMachineCertsCurrentUser.Open(OpenFlags.ReadWrite); - localMachineCertsCurrentUser.RemoveRange(kvc.LocalCerts.Certificates); - localMachineCertsCurrentUser.AddRange(kvc.KeyVaultCertificates); - localMachineCertsCurrentUser.Close(); - updated = true; - } - if(!updated) - { localMachineCerts.RemoveRange(kvc.LocalCerts.Certificates); localMachineCerts.AddRange(kvc.KeyVaultCertificates); } } } + catch (Exception ex) + { + Console.WriteLine("Failed to rotate certificates"); + Console.WriteLine(ex.Message); + Console.WriteLine(ex.StackTrace); + } using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser, OpenFlags.ReadWrite)) { diff --git a/src/tools/CertHelperTests/KeyVaultCertTests.cs b/src/tools/CertHelperTests/KeyVaultCertTests.cs index b56e59a304a..a761ff00575 100644 --- a/src/tools/CertHelperTests/KeyVaultCertTests.cs +++ b/src/tools/CertHelperTests/KeyVaultCertTests.cs @@ -44,52 +44,43 @@ private static void CertStoreSetup(out Mock mockTokenCred, out mockCertClient = new Mock(new Uri("https://dotnetperfkeyvault.vault.azure.net/"), mockTokenCred.Object); mockSecretClient = new Mock(new Uri("https://dotnetperfkeyvault.vault.azure.net/"), mockTokenCred.Object); KeyVaultCertificateWithPolicy? mockCert1, mockCert2; - X509Certificate2 cert1, cert2; - MakeCerts(out mockCert1, out mockCert2, out cert1, out cert2, localAndKeyVaultDifferent); + X509Certificate2 cert1, cert2, cert3; + MakeCerts(out mockCert1, out mockCert2, out cert1, out cert2, out cert3, localAndKeyVaultDifferent); var certCollection = new X509Certificate2Collection { cert1, cert2 }; -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. - mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert1Name, default)).ReturnsAsync(Response.FromValue(mockCert1, null)); -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert1Name, default)).ReturnsAsync(Response.FromValue(mockCert1, null!)); if (missingKeyVaultCerts) { -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. - mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert2Name, default)).ReturnsAsync(Response.FromValue(null, null)); -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert2Name, default)).ReturnsAsync(Response.FromValue(null!, null!)); } else { -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. - mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert2Name, default)).ReturnsAsync(Response.FromValue(mockCert2, null)); -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert2Name, default)).ReturnsAsync(Response.FromValue(mockCert2, null!)); } //var secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(certCollection[0].Export(X509ContentType.Cert))); //var secret2 = new KeyVaultSecret(Constants.Cert2Name, Convert.ToBase64String(certCollection[1].Export(X509ContentType.Cert))); -#pragma warning disable CS8602 // Dereference of a possibly null reference. - var secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(cert1.Export(X509ContentType.Pfx))); -#pragma warning restore CS8602 // Dereference of a possibly null reference. -#pragma warning disable CS8602 // Dereference of a possibly null reference. - var secret2 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(cert1.Export(X509ContentType.Pfx))); -#pragma warning restore CS8602 // Dereference of a possibly null reference. - -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. -#pragma warning disable CS8602 // Dereference of a possibly null reference. - mockSecretClient.Setup(s => s.GetSecretAsync(Constants.Cert1Name, mockCert1.SecretId.Segments.Last(), default)).ReturnsAsync(Response.FromValue(secret1, null)); -#pragma warning restore CS8602 // Dereference of a possibly null reference. -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. -#pragma warning disable CS8625 // Cannot convert null literal to non-nullable reference type. -#pragma warning disable CS8602 // Dereference of a possibly null reference. - mockSecretClient.Setup(s => s.GetSecretAsync(Constants.Cert2Name, mockCert2.SecretId.Segments.Last(), default)).ReturnsAsync(Response.FromValue(secret2, null)); -#pragma warning restore CS8602 // Dereference of a possibly null reference. -#pragma warning restore CS8625 // Cannot convert null literal to non-nullable reference type. + + KeyVaultSecret secret1; + if(localAndKeyVaultDifferent) + { + secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(cert3.Export(X509ContentType.Pfx))); + } + else + { + secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(cert1.Export(X509ContentType.Pfx))); + } + var secret2 = new KeyVaultSecret(Constants.Cert2Name, Convert.ToBase64String(cert2.Export(X509ContentType.Pfx))); + + mockSecretClient.Setup(s => s.GetSecretAsync(Constants.Cert1Name, mockCert1!.SecretId.Segments.Last(), default)).ReturnsAsync(Response.FromValue(secret1, null!)); + mockSecretClient.Setup(s => s.GetSecretAsync(Constants.Cert2Name, mockCert2!.SecretId.Segments.Last(), default)).ReturnsAsync(Response.FromValue(secret2, null!)); mockLocalCert = new Mock(); mockLocalCert.Setup(lc => lc.Certificates).Returns(certCollection); } - private static void MakeCerts(out KeyVaultCertificateWithPolicy? mockCert1, out KeyVaultCertificateWithPolicy? mockCert2, out X509Certificate2 cert1, out X509Certificate2 cert2, bool localAndKeyVaultDifferent = false) + private static void MakeCerts(out KeyVaultCertificateWithPolicy? mockCert1, out KeyVaultCertificateWithPolicy? mockCert2, out X509Certificate2 cert1, out X509Certificate2 cert2, out X509Certificate2 cert3, bool localAndKeyVaultDifferent = false) { using var rsa1 = RSA.Create(); // generate asymmetric key pair var req1 = new CertificateRequest("cn=perflabtest", rsa1, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); @@ -97,15 +88,16 @@ private static void MakeCerts(out KeyVaultCertificateWithPolicy? mockCert1, out using var rsa2 = RSA.Create(); // generate asymmetric key pair var req2 = new CertificateRequest("cn=perflabtest", rsa2, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); var tmpCert2 = req2.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); - cert1 = tmpCert1; - cert2 = tmpCert2; + cert1 = new X509Certificate2(tmpCert1); + cert2 = new X509Certificate2(tmpCert2); if(localAndKeyVaultDifferent) { - var ecdsa = ECDsa.Create(); // generate asymmetric key pair - var req = new CertificateRequest("cn=perflabtest", ecdsa, HashAlgorithmName.SHA256); - tmpCert1 = X509CertificateLoader.LoadPkcs12(req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)).Export(X509ContentType.Pkcs12), ""); + using var rsa3 = RSA.Create(); // generate asymmetric key pair + var req = new CertificateRequest("cn=perflabtest", rsa3, HashAlgorithmName.SHA256, RSASignaturePadding.Pkcs1); + tmpCert1 = req.CreateSelfSigned(DateTimeOffset.Now, DateTimeOffset.Now.AddYears(5)); } + cert3 = new X509Certificate2(tmpCert1); mockCert1 = CertificateModelFactory.KeyVaultCertificateWithPolicy(CertificateModelFactory.CertificateProperties(Constants.Cert1Id, Constants.Cert1Name, x509thumbprint: Convert.FromHexString(tmpCert1.Thumbprint)), Constants.Cert1Id, Constants.Cert1Id, tmpCert1.GetRawCertData()); From d91b815fae277ef65bf33568e31abeb2e4c6d49c Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 16 Jan 2025 15:50:07 -0800 Subject: [PATCH 18/24] Update moq version --- src/tools/CertHelperTests/CertRotatorTests.csproj | 2 +- src/tools/CertHelperTests/KeyVaultCertTests.cs | 3 --- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/tools/CertHelperTests/CertRotatorTests.csproj b/src/tools/CertHelperTests/CertRotatorTests.csproj index 3acaadb46a9..0ea831f1d18 100644 --- a/src/tools/CertHelperTests/CertRotatorTests.csproj +++ b/src/tools/CertHelperTests/CertRotatorTests.csproj @@ -15,7 +15,7 @@ runtime; build; native; contentfiles; analyzers; buildtransitive - + all diff --git a/src/tools/CertHelperTests/KeyVaultCertTests.cs b/src/tools/CertHelperTests/KeyVaultCertTests.cs index a761ff00575..426b52df5a5 100644 --- a/src/tools/CertHelperTests/KeyVaultCertTests.cs +++ b/src/tools/CertHelperTests/KeyVaultCertTests.cs @@ -59,9 +59,6 @@ private static void CertStoreSetup(out Mock mockTokenCred, out mockCertClient.Setup(c => c.GetCertificateAsync(Constants.Cert2Name, default)).ReturnsAsync(Response.FromValue(mockCert2, null!)); } - //var secret1 = new KeyVaultSecret(Constants.Cert1Name, Convert.ToBase64String(certCollection[0].Export(X509ContentType.Cert))); - //var secret2 = new KeyVaultSecret(Constants.Cert2Name, Convert.ToBase64String(certCollection[1].Export(X509ContentType.Cert))); - KeyVaultSecret secret1; if(localAndKeyVaultDifferent) { From d35f87bf5cf811a2a98ee520f018ac2f8d0a9452 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 16 Jan 2025 15:51:31 -0800 Subject: [PATCH 19/24] Remove testing changes --- azure-pipelines.yml | 350 ++++++++++++++++++++++---------------------- 1 file changed, 175 insertions(+), 175 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ea3a30918d0..9f254cd0809 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -459,135 +459,135 @@ jobs: # Scheduled runs will run all of the jobs on the PerfTigers, as well as the Arm64 job - ${{ if or(and(and(ne(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'Schedule')), not(contains(variables['Build.QueuedBy'], 'Weekly'))), parameters.runScheduledPrivateJobs) }}: - # # SDK scenario benchmarks - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/scenarios.yml - # buildMachines: - # - win-x64 - # - win-x86 - # #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available - # isPublic: false - # jobParameters: - # kind: sdk_scenarios - # projectFile: sdk_scenarios.proj - # channels: - # - main - # - 8.0 + # SDK scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + - win-x86 + #- ubuntu-x64-1804 reenable under new machine on new ubuntu once lttng/events are available + isPublic: false + jobParameters: + kind: sdk_scenarios + projectFile: sdk_scenarios.proj + channels: + - main + - 8.0 - # # Blazor 3.2 scenario benchmarks - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/scenarios.yml - # buildMachines: - # - win-x64 - # isPublic: false - # jobParameters: - # kind: blazor_scenarios - # projectFile: blazor_scenarios.proj - # channels: - # - main - # - 8.0 - - # # F# benchmarks - # - ${{ if false }}: # skipping, no useful benchmarks there currently - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: fsharp - # csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj - # runCategories: 'fsharp' - # channels: - # - main - # - 8.0 + # Blazor 3.2 scenario benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/scenarios.yml + buildMachines: + - win-x64 + isPublic: false + jobParameters: + kind: blazor_scenarios + projectFile: blazor_scenarios.proj + channels: + - main + - 8.0 - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: fsharpmicro - # csproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj - # runCategories: 'FSharpMicro' - # channels: - # - main - # - 8.0 + # F# benchmarks + - ${{ if false }}: # skipping, no useful benchmarks there currently + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: fsharp + csproj: src\benchmarks\real-world\FSharp\FSharp.fsproj + runCategories: 'fsharp' + channels: + - main + - 8.0 - # # bepuphysics benchmarks - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: bepuphysics - # csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj - # runCategories: 'BepuPhysics' - # channels: - # - main - # - 8.0 + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: fsharpmicro + csproj: src\benchmarks\micro-fsharp\MicrobenchmarksFSharp.fsproj + runCategories: 'FSharpMicro' + channels: + - main + - 8.0 - # # ImageSharp benchmarks - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: imagesharp - # csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj - # runCategories: 'ImageSharp' - # channels: - # - main - # - 8.0 + # bepuphysics benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: bepuphysics + csproj: src\benchmarks\real-world\bepuphysics2\DemoBenchmarks.csproj + runCategories: 'BepuPhysics' + channels: + - main + - 8.0 - # # Akade.IndexedSet benchmarks - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: akadeindexedset - # csproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj - # runCategories: 'AkadeIndexedSet' - # channels: - # - main + # ImageSharp benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: imagesharp + csproj: src\benchmarks\real-world\ImageSharp\ImageSharp.Benchmarks.csproj + runCategories: 'ImageSharp' + channels: + - main + - 8.0 - # ML.NET benchmarks + # Akade.IndexedSet benchmarks - template: /eng/performance/build_machine_matrix.yml parameters: jobTemplate: /eng/performance/benchmark_jobs.yml buildMachines: - # - win-x64 - # - ubuntu-x64 + - win-x64 + - ubuntu-x64 + - win-arm64 + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: akadeindexedset + csproj: src\benchmarks\real-world\Akade.IndexedSet.Benchmarks\Akade.IndexedSet.Benchmarks.csproj + runCategories: 'AkadeIndexedSet' + channels: + - main + + ML.NET benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 - win-arm64 - # - win-arm64-ampere - # - ubuntu-arm64-ampere + - win-arm64-ampere + - ubuntu-arm64-ampere isPublic: false jobParameters: kind: mlnet @@ -602,65 +602,65 @@ jobs: - DOTNET_GCHeapCount=4 - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - # # Roslyn benchmarks - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # - win-arm64 - # - win-arm64-ampere - # - ubuntu-arm64-ampere - # isPublic: false - # jobParameters: - # kind: roslyn - # csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj - # runCategories: 'roslyn' - # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - # - main - # - 8.0 - # affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account - # runEnvVars: - # - DOTNET_GCgen0size=410000 # ~4MB - # - DOTNET_GCHeapCount=4 - # - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB + # Roslyn benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + - win-arm64 + - win-arm64-ampere + - ubuntu-arm64-ampere + isPublic: false + jobParameters: + kind: roslyn + csproj: src\benchmarks\real-world\Roslyn\CompilerBenchmarks.csproj + runCategories: 'roslyn' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 8.0 + affinity: '85' # (01010101) Enables alternating process threads to take hyperthreading into account + runEnvVars: + - DOTNET_GCgen0size=410000 # ~4MB + - DOTNET_GCHeapCount=4 + - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - # ILLink benchmarks - # disabled because of: https://github.com/dotnet/performance/issues/3569 - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # - win-x64 - # - ubuntu-x64 - # # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) - # isPublic: false - # jobParameters: - # kind: illink - # csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - # runCategories: 'illink' - # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - # - main - # - 8.0 + ILLink benchmarks + disabled because of: https://github.com/dotnet/performance/issues/3569 + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + - win-x64 + - ubuntu-x64 + # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) + isPublic: false + jobParameters: + kind: illink + csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + runCategories: 'illink' + channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + - main + - 8.0 - # # Powershell benchmarks - # - template: /eng/performance/build_machine_matrix.yml - # parameters: - # jobTemplate: /eng/performance/benchmark_jobs.yml - # buildMachines: - # # - win-x64 - # # - ubuntu-x64 - # - win-arm64 - # # - ubuntu-arm64 - # isPublic: false - # jobParameters: - # kind: powershell - # csproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj - # runCategories: 'Public Internal' - # channels: - # - main - # - 8.0 + # Powershell benchmarks + - template: /eng/performance/build_machine_matrix.yml + parameters: + jobTemplate: /eng/performance/benchmark_jobs.yml + buildMachines: + # - win-x64 + # - ubuntu-x64 + - win-arm64 + # - ubuntu-arm64 + isPublic: false + jobParameters: + kind: powershell + csproj: src\benchmarks\real-world\PowerShell.Benchmarks\PowerShell.Benchmarks.csproj + runCategories: 'Public Internal' + channels: + - main + - 8.0 # Secret Sync - job: Synchronize From 59fb1cb364cfeb2f1a18839326194429ba19e854 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 16 Jan 2025 15:52:50 -0800 Subject: [PATCH 20/24] Remove a few more testing changes --- azure-pipelines.yml | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9f254cd0809..9b004cef0ba 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -578,7 +578,7 @@ jobs: channels: - main - ML.NET benchmarks + # ML.NET benchmarks - template: /eng/performance/build_machine_matrix.yml parameters: jobTemplate: /eng/performance/benchmark_jobs.yml @@ -626,23 +626,23 @@ jobs: - DOTNET_GCHeapCount=4 - DOTNET_GCTotalPhysicalMemory=400000000 # 16GB - ILLink benchmarks - disabled because of: https://github.com/dotnet/performance/issues/3569 - - template: /eng/performance/build_machine_matrix.yml - parameters: - jobTemplate: /eng/performance/benchmark_jobs.yml - buildMachines: - - win-x64 - - ubuntu-x64 - # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) - isPublic: false - jobParameters: - kind: illink - csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj - runCategories: 'illink' - channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only - - main - - 8.0 + # ILLink benchmarks + # disabled because of: https://github.com/dotnet/performance/issues/3569 + # - template: /eng/performance/build_machine_matrix.yml + # parameters: + # jobTemplate: /eng/performance/benchmark_jobs.yml + # buildMachines: + # - win-x64 + # - ubuntu-x64 + # # Illink.Utilities is not supported on ARM: The type initializer for 'ILLinkBenchmarks.Utilities' threw a NotSupportedException (Unsupported architecture). (06/2023) + # isPublic: false + # jobParameters: + # kind: illink + # csproj: src\benchmarks\real-world\ILLink\ILLinkBenchmarks.csproj + # runCategories: 'illink' + # channels: # for Roslyn jobs we want to check .NET Core 3.1 and 5.0 only + # - main + # - 8.0 # Powershell benchmarks - template: /eng/performance/build_machine_matrix.yml From 7d8f191f48305c7021a43d4a5ccb2dfe858bf398 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 16 Jan 2025 15:53:28 -0800 Subject: [PATCH 21/24] Remove last testing change --- azure-pipelines.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 9b004cef0ba..6a6593bb439 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -649,10 +649,10 @@ jobs: parameters: jobTemplate: /eng/performance/benchmark_jobs.yml buildMachines: - # - win-x64 - # - ubuntu-x64 + - win-x64 + - ubuntu-x64 - win-arm64 - # - ubuntu-arm64 + - ubuntu-arm64 isPublic: false jobParameters: kind: powershell From 5cb322c738a39c03f0d6d25d07a60f63be942aff Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 16 Jan 2025 16:36:27 -0800 Subject: [PATCH 22/24] Add simple logging --- src/tools/CertHelper/CertHelper.csproj | 9 ++++---- src/tools/CertHelper/Program.cs | 29 +++++++++++++++++++++++++- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/tools/CertHelper/CertHelper.csproj b/src/tools/CertHelper/CertHelper.csproj index 185bb4c2ef8..005a2ceb892 100644 --- a/src/tools/CertHelper/CertHelper.csproj +++ b/src/tools/CertHelper/CertHelper.csproj @@ -10,10 +10,11 @@ - - - - + + + + + diff --git a/src/tools/CertHelper/Program.cs b/src/tools/CertHelper/Program.cs index 8ff23675bc1..94c5f2cab42 100644 --- a/src/tools/CertHelper/Program.cs +++ b/src/tools/CertHelper/Program.cs @@ -1,10 +1,15 @@ -using System.Security.Cryptography.X509Certificates; +using Azure.Identity; +using Azure.Storage.Blobs; +using Azure.Storage.Blobs.Specialized; +using System.Security.Cryptography.X509Certificates; using System.Text; namespace CertHelper; internal class Program { + static readonly string TENANT_ID = "72f988bf-86f1-41af-91ab-2d7cd011db47"; + static readonly string CERT_CLIENT_ID = "8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc"; static async Task Main(string[] args) { try @@ -20,6 +25,26 @@ static async Task Main(string[] args) localMachineCerts.AddRange(kvc.KeyVaultCertificates); } } + var bcc = new BlobContainerClient(new Uri("https://pvscmdupload.blob.core.windows.net/certstatus"), + new ClientCertificateCredential(TENANT_ID, CERT_CLIENT_ID, kvc.KeyVaultCertificates.First())); + var currentKeyValutCertThumbprints = ""; + foreach(var cert in kvc.KeyVaultCertificates) + { + currentKeyValutCertThumbprints += $"[{DateTimeOffset.UtcNow}] {cert.Thumbprint}{Environment.NewLine}"; + } + var blob = bcc.GetBlobClient(System.Environment.MachineName); + if (blob.Exists()) + { + var result = blob.DownloadContent(); + var currentBlob = result.Value.Content.ToString(); + currentBlob = currentBlob + currentKeyValutCertThumbprints; + blob.Upload(new MemoryStream(Encoding.UTF8.GetBytes(currentBlob)), overwrite: true); + } + else + { + blob.Upload(new MemoryStream(Encoding.UTF8.GetBytes(currentKeyValutCertThumbprints)), overwrite: false); + } + } catch (Exception ex) { @@ -28,6 +53,8 @@ static async Task Main(string[] args) Console.WriteLine(ex.StackTrace); } + + using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser, OpenFlags.ReadWrite)) { foreach(var cert in store.Certificates.Find(X509FindType.FindBySubjectName, "dotnetperf.microsoft.com", false)) From c735705979f71a0bce848f224bc57e037526d818 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Thu, 16 Jan 2025 19:28:01 -0800 Subject: [PATCH 23/24] Address pr feedback --- src/tools/CertHelper/KeyVaultCert.cs | 10 +++++----- src/tools/CertHelper/Program.cs | 2 +- src/tools/CertHelperTests/KeyVaultCertTests.cs | 12 ++++++------ 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/tools/CertHelper/KeyVaultCert.cs b/src/tools/CertHelper/KeyVaultCert.cs index d72a652b8c7..32d3a21d7d1 100644 --- a/src/tools/CertHelper/KeyVaultCert.cs +++ b/src/tools/CertHelper/KeyVaultCert.cs @@ -34,12 +34,12 @@ public KeyVaultCert(TokenCredential? cred = null, CertificateClient? certClient KeyVaultCertificates = new X509Certificate2Collection(); } - public async Task GetKeyVaultCerts() + public async Task LoadKeyVaultCertsAsync() { - KeyVaultCertificates.Add(await FindCertificateInKeyVault(Constants.Cert1Name)); - KeyVaultCertificates.Add(await FindCertificateInKeyVault(Constants.Cert2Name)); + KeyVaultCertificates.Add(await FindCertificateInKeyVaultAsync(Constants.Cert1Name)); + KeyVaultCertificates.Add(await FindCertificateInKeyVaultAsync(Constants.Cert2Name)); - if (KeyVaultCertificates.Count != 2 || KeyVaultCertificates.Where(c => c == null).Count() > 0) + if (KeyVaultCertificates.Where(c => c == null).Count() > 0) { throw new Exception("One or more certificates not found"); } @@ -70,7 +70,7 @@ private async Task GetCertifcateCredentialAsync(str return ccc; } - private async Task FindCertificateInKeyVault(string certName) + private async Task FindCertificateInKeyVaultAsync(string certName) { var keyVaultCert = await CertClient.GetCertificateAsync(certName); if(keyVaultCert.Value == null) diff --git a/src/tools/CertHelper/Program.cs b/src/tools/CertHelper/Program.cs index 94c5f2cab42..71563a205e0 100644 --- a/src/tools/CertHelper/Program.cs +++ b/src/tools/CertHelper/Program.cs @@ -15,7 +15,7 @@ static async Task Main(string[] args) try { var kvc = new KeyVaultCert(); - await kvc.GetKeyVaultCerts(); + await kvc.LoadKeyVaultCertsAsync(); if (kvc.ShouldRotateCerts()) { using (var localMachineCerts = new X509Store(StoreName.My, StoreLocation.CurrentUser)) diff --git a/src/tools/CertHelperTests/KeyVaultCertTests.cs b/src/tools/CertHelperTests/KeyVaultCertTests.cs index 426b52df5a5..7f9621352b4 100644 --- a/src/tools/CertHelperTests/KeyVaultCertTests.cs +++ b/src/tools/CertHelperTests/KeyVaultCertTests.cs @@ -18,7 +18,7 @@ namespace CertHelper.Tests; public class KeyVaultCertTests { [Fact] - public async Task GetKeyVaultCerts_ShouldAddCertificatesToCollection() + public async Task LoadKeyVaultCertsAsync_ShouldAddCertificatesToCollection() { // Arrange Mock mockTokenCred; @@ -30,7 +30,7 @@ public async Task GetKeyVaultCerts_ShouldAddCertificatesToCollection() var keyVaultCert = new KeyVaultCert(mockTokenCred.Object, mockCertClient.Object, mockSecretClient.Object, mockLocalCert.Object); // Act - await keyVaultCert.GetKeyVaultCerts(); + await keyVaultCert.LoadKeyVaultCertsAsync(); // Assert Assert.Equal(2, keyVaultCert.KeyVaultCertificates.Count); @@ -103,7 +103,7 @@ private static void MakeCerts(out KeyVaultCertificateWithPolicy? mockCert1, out } [Fact] - public async Task GetKeyVaultCerts_ShouldThrowException_WhenCertificatesNotFound() + public async Task LoadKeyVaultCertsAsync_ShouldThrowException_WhenCertificatesNotFound() { // Arrange Mock mockTokenCred; @@ -115,7 +115,7 @@ public async Task GetKeyVaultCerts_ShouldThrowException_WhenCertificatesNotFound var keyVaultCert = new KeyVaultCert(mockTokenCred.Object, mockCertClient.Object, mockSecretClient.Object, mockLocalCert.Object); // Act & Assert - await Assert.ThrowsAsync(() => keyVaultCert.GetKeyVaultCerts()); + await Assert.ThrowsAsync(() => keyVaultCert.LoadKeyVaultCertsAsync()); } [Fact] @@ -131,7 +131,7 @@ public async Task ShouldRotateCerts_ShouldReturnTrue_WhenThumbprintsDoNotMatch() var keyVaultCert = new KeyVaultCert(mockTokenCred.Object, mockCertClient.Object, mockSecretClient.Object, mockLocalCert.Object); // Act - await keyVaultCert.GetKeyVaultCerts(); + await keyVaultCert.LoadKeyVaultCertsAsync(); var result = keyVaultCert.ShouldRotateCerts(); @@ -152,7 +152,7 @@ public async Task ShouldRotateCerts_ShouldReturnFalse_WhenThumbprintsMatch() var keyVaultCert = new KeyVaultCert(mockTokenCred.Object, mockCertClient.Object, mockSecretClient.Object, mockLocalCert.Object); // Act - await keyVaultCert.GetKeyVaultCerts(); + await keyVaultCert.LoadKeyVaultCertsAsync(); var result = keyVaultCert.ShouldRotateCerts(); From 2ce7bbb1d74b16cf8db9f9139cba16080bde0333 Mon Sep 17 00:00:00 2001 From: Drew Scoggins Date: Fri, 17 Jan 2025 15:43:54 -0800 Subject: [PATCH 24/24] Address PR feedback --- src/tools/CertHelper/KeyVaultCert.cs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/src/tools/CertHelper/KeyVaultCert.cs b/src/tools/CertHelper/KeyVaultCert.cs index 32d3a21d7d1..067e9025e94 100644 --- a/src/tools/CertHelper/KeyVaultCert.cs +++ b/src/tools/CertHelper/KeyVaultCert.cs @@ -15,22 +15,22 @@ namespace CertHelper; public class KeyVaultCert { - private readonly string KeyVaultUrl = "https://dotnetperfkeyvault.vault.azure.net/"; - private readonly string TenantId = "72f988bf-86f1-41af-91ab-2d7cd011db47"; - private readonly string ClientId = "8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc"; + private readonly string _keyVaultUrl = "https://dotnetperfkeyvault.vault.azure.net/"; + private readonly string _tenantId = "72f988bf-86f1-41af-91ab-2d7cd011db47"; + private readonly string _clientId = "8c4b65ef-5a73-4d5a-a298-962d4a4ef7bc"; public X509Certificate2Collection KeyVaultCertificates { get; set; } public ILocalCert LocalCerts { get; set; } - private TokenCredential Credential { get; set; } - private CertificateClient CertClient { get; set; } - private SecretClient SecretClient { get; set; } + private TokenCredential _credential { get; set; } + private CertificateClient _certClient { get; set; } + private SecretClient _secretClient { get; set; } public KeyVaultCert(TokenCredential? cred = null, CertificateClient? certClient = null, SecretClient? secretClient = null, ILocalCert? localCerts = null) { LocalCerts = localCerts ?? new LocalCert(); - Credential = cred ?? GetCertifcateCredentialAsync(TenantId, ClientId, LocalCerts.Certificates).Result; - CertClient = certClient ?? new CertificateClient(new Uri(KeyVaultUrl), Credential); - SecretClient = secretClient ?? new SecretClient(new Uri(KeyVaultUrl), Credential); + _credential = cred ?? GetCertifcateCredentialAsync(_tenantId, _clientId, LocalCerts.Certificates).Result; + _certClient = certClient ?? new CertificateClient(new Uri(_keyVaultUrl), _credential); + _secretClient = secretClient ?? new SecretClient(new Uri(_keyVaultUrl), _credential); KeyVaultCertificates = new X509Certificate2Collection(); } @@ -72,12 +72,12 @@ private async Task GetCertifcateCredentialAsync(str private async Task FindCertificateInKeyVaultAsync(string certName) { - var keyVaultCert = await CertClient.GetCertificateAsync(certName); + var keyVaultCert = await _certClient.GetCertificateAsync(certName); if(keyVaultCert.Value == null) { throw new Exception("Certificate not found in Key Vault"); } - var secret = await SecretClient.GetSecretAsync(keyVaultCert.Value.Name, keyVaultCert.Value.SecretId.Segments.Last()); + var secret = await _secretClient.GetSecretAsync(keyVaultCert.Value.Name, keyVaultCert.Value.SecretId.Segments.Last()); if(secret.Value == null) { throw new Exception("Certificate secret not found in Key Vault");