SERVER-101468 Update certificate generation to be cross-platform and deterministic (#33814)

GitOrigin-RevId: 3ef1af57ac423e448ee1a7d7d9a21a6fad0ae79f

Author: marksg07

.github/CODEOWNERS

@@ -2636,3 +2636,6 @@ WORKSPACE.bazel @10gen/devprod-build @svc-auto-approve-bot
 
 # The following patterns are parsed from ./src/third_party/libmongocrypt/OWNERS.yml
 /src/third_party/libmongocrypt/**/* @10gen/server-security @svc-auto-approve-bot
+
+# The following patterns are parsed from ./x509/OWNERS.yml
+/x509/**/* @10gen/server-security @svc-auto-approve-bot

etc/evergreen_yml_components/tasks/compile_tasks_shared.yml

@@ -222,6 +222,7 @@ tasks:
             - "src/src/third_party/schemastore.org/**"
             - "src/poetry.lock"
             - "src/pyproject.toml"
+            - "src/x509/**"
           exclude_files:
             - "src/*_test.pdb"
 

jstests/noPassthrough/x509/BUILD.bazel

@@ -0,0 +1,13 @@
+load("@aspect_rules_js//js:defs.bzl", "js_library")
+
+js_library(
+    name = "all_javascript_files",
+    srcs = glob([
+        "*.js",
+    ]),
+    target_compatible_with = select({
+        "//bazel/config:ppc_or_s390x": ["@platforms//:incompatible"],
+        "//conditions:default": [],
+    }),
+    visibility = ["//visibility:public"],
+)

jstests/noPassthrough/x509/mkcert_is_deterministic.js

@@ -0,0 +1,52 @@
+// Test that mkcert.py generates certificates deterministically, and that pkcs12 certificates, while
+// not deterministic, can be generated.
+import {getPython3Binary} from "jstests/libs/python.js";
+
+const now = new Date();
+// Note - month is 0-indexed, so 11 is December.
+if (now.getMonth() == 11 && now.getDate() == 31 && now.getHours() == 23) {
+    jsTestLog(
+        "Deterministic certificate generation relies on the current year being constant; skipping test as there is less than an hour until the year changes.");
+    quit();
+}
+
+// Print the openssl version to help with debugging.
+clearRawMongoProgramOutput();
+assert.eq(runNonMongoProgram("openssl", "version"), 0);
+jsTest.log(rawMongoProgramOutput(".*"));
+
+const basedir = MongoRunner.dataPath + "certs/";
+const genpath = basedir + "generated/";
+mkdir(genpath);
+
+// Run mkcert, and ensure it succeeds and that expected results are generated successfully.
+jsTest.log("Running mkcert");
+clearRawMongoProgramOutput();
+let res = runNonMongoProgram(getPython3Binary(), "-m", "x509.mkcert", "--mkcrl", "-o", genpath);
+jsTest.log(rawMongoProgramOutput(".*"));
+assert.eq(res, 0);
+assert(fileExists(genpath + "ca.pem"));
+assert(fileExists(genpath + "crl.pem"));
+
+// Run mkcert again, to a different path.
+const genpath2 = basedir + "generated2/";
+mkdir(genpath2);
+jsTest.log("Running mkcert again");
+res = runNonMongoProgram(getPython3Binary(), "-m", "x509.mkcert", "--mkcrl", "-o", genpath2);
+assert.eq(res, 0);
+
+// Diff the two generation paths to make sure the contents of the paths are identical.
+jsTest.log("Running diff");
+clearRawMongoProgramOutput();
+res = runNonMongoProgram("diff", "-r", genpath, genpath2);
+assert.eq(res, 0);
+const diffout = rawMongoProgramOutput(".*").trim();
+assert.eq("", diffout, diffout);
+
+// Run mkcert on the apple-certs.yml definitions file, which contains pkcs12 certificates, and
+// ensure a .pfx file was generated.
+jsTest.log("Running apple certs");
+res = runNonMongoProgram(
+    getPython3Binary(), "-m", "x509.mkcert", "-o", genpath, "--config", "x509/apple-certs.yml");
+assert.eq(res, 0);
+assert(fileExists(genpath + "macos-trusted-server.pfx"));

poetry.lock

@@ -145,6 +145,8 @@ shrub-py = "^3.1.4"
 ocspresponder = "^0.5.0"
 flask = "^2.3.3"
 ocspbuilder = "^0.10.2"
+ecdsa = "^0.19.0"
+asn1crypto = "^1.5.1"
 toml = ">=0.10.2,<0.11.0"
 
 # Werkzeug is needed for ocsp tests in ocsp_mock.py

pyproject.toml

@@ -0,0 +1,5 @@
+version: 1.0.0
+filters:
+  - "*":
+    approvers:
+      - 10gen/server-security

x509/OWNERS.yml

@@ -0,0 +1,93 @@
+This directory contains:
+
+- mkcert.py
+  Python script; uses the cryptography package to deterministically generate X509 certificates,
+  CRLs, and digests based on the contents of certs.yml.
+- certs.yml
+  Main certificate definition file.
+- apple-certs.yml
+  Certificate definitions for certs to be installed on provision of OSX machines.
+
+To run:
+
+python -m x509.mkcert [--config CONFIG] [--mkcrl | --no-mkcrl] [-o OUTPUT] [--static-dir STATIC_DIR] [certs ...]
+- CONFIG is the path to the certs.yml file specifying a list of certificates, default x509/certs.yml
+- OUTPUT is the path to a directory where the generated items will be stored, default .
+- STATIC_DIR is the path where signing keys needed by certificates are stored, default x509/static 
+- If --mkcrl is specified, CRLs will be generated after certificate generation ends. Default false.
+  These are hardcoded and require certain certificates to be generated to work.
+- certs is an optional list of certificate names to generate. If it is not specified, all
+  certificates specified in the config are generated.
+
+If a certificate specified in certs references other certificates, its dependencies and
+subdependencies will be generated.
+
+Deterministic generation is based on the current year. This means that if mkcert is run on the same
+definitions file twice, it will produce the same certificates if both runs were in the same year.
+One exception to this is the pkcs12 format; the cryptography library does not provide
+functionality for generating pkcs12 bundles deterministically.
+
+Future work:
+
+- Define CRLs in the definition file instead of hardcoding them.
+- Define keys in the definition file, and make a script to generate all necessary keys which would
+  be run whenever a new key was defined.
+
+certs.yml format:
+
+global: # Optional, default value to use for Key1 for all certs, overridden by values in cert entries.
+  Key1: Value1
+  ...
+
+certs:
+    # Required, this will be used as the name of the file, and for referencing issuers.
+  - name: 'name-of-cert.pem'
+    # Required, this will be included in the header of the generated certificate.
+    description: Tell us about yourself.
+    # Required, The X509 subject name.
+    Subject: { C: US, ST: New York, etc... }
+    # Required, Who is the (intermediate) CA for this certificate.  May be 'self'.
+    Issuer: 'ca.pem'
+    # Required, relative (within static directory) path to the keyfile to sign this certificate with.
+    keyfile: 'key.pem'
+    # Optional, set to true to ignore global.Subject values.
+    explicit_subject: false
+    # Optional, serial number to assign this certificate (default: sequential numbers starting from 1000)
+    serial: 42
+    # Optional, validity start date, expressed in seconds relative to midnight on the first day of the current year.
+    not_before: -86400 # 1 day before 
+    # Optional, validity end date, currently expressed in seconds relative to midnight on the first day of the current year.
+    # Note that not_after - not_before, the validity period, should be less than or equal to 825 days, see:
+    # https://support.apple.com/en-us/HT210176
+    not_after: 71107200 # 823 days after
+    # Optional, IDs of other public keys to append to the file
+    append_certs: ['ca.pem', 'intermediate-ca.pem', ...]
+    # Optional, passphrase to encript private key with
+    passphrase: 'secret'
+    # Optional, make a pkcs12 copy of the certificate
+    pkcs12: true | map with keys below
+        # Optional, all PKCS#12 keys must be encrypted. Will use cert.passphase if not provided.
+        passphrase: 'secret'
+        # Optional, name of PKCS#12 version of certificate. If not provided, the original cert will be overwritten with the PKCS#12 version
+        name: 'name-of-cert.pfx'
+    # Optional, in addition to the .pem file, write just the certificate to a .crt file and just the signing key to a .key file
+    split_cert_and_key: true
+    # Optional, don't write a header comment to this cert
+    include_header: false
+    # Optional, X.509 extensions to include in the certificate
+    extensions: # All extensions are optional.
+    - basicConstraints: {}
+    - keyUsage: {}
+    - extendedKeyUsage: {}
+    - subjectAltName: {DNS: [...], IP: [...]}
+    - subjectKeyIdentifier: hash
+    - authorityKeyIdentifier: keyid | issuer
+    - authorityInfoAccess: 
+      - method: OCSP
+      - location: uri-to-OCSP-server
+    - mustStaple: true
+    - nsComment: "Comment"
+    - mongoRoles:
+      - {role: readWrite, db: test1}
+      - {role: read, db: test2}
+    - mongoClusterMembership: clusterName

x509/README

@@ -0,0 +1,49 @@
+# Definition for testing certificates to be created and installed into the MacOS trusted keychain upon provision.
+# On MacOS provision, these certificates will be written to /opt/x509.
+
+global:
+  Subject:
+    C: "US"
+    ST: "New York"
+    L: "New York City"
+    O: "MongoDB"
+    OU: "Kernel"
+  keyfile: "macos_key.pem"
+
+certs:
+  - name: "macos-trusted-ca.pem"
+    description: CA for trusted MacOS client/server certificate chain.
+    Subject: {CN: "Trusted MacOS Kernel Test CA"}
+    Issuer: self
+    keyfile: "macos_ca_key.pem"
+    extensions:
+      basicConstraints: {CA: true}
+      subjectAltName:
+        DNS: localhost
+        IP: 127.0.0.1
+
+  - name: "macos-trusted-client.pem"
+    description: Client certificate for trusted MacOS chain.
+    Subject: {CN: "Trusted MacOS Kernel Test Client"}
+    Issuer: "macos-trusted-ca.pem"
+    pkcs12:
+      passphrase: "qwerty"
+      name: "macos-trusted-client.pfx"
+    extensions:
+      extendedKeyUsage: [clientAuth]
+      subjectAltName:
+        DNS: localhost
+        IP: 127.0.0.1
+
+  - name: "macos-trusted-server.pem"
+    description: Server certificate for trusted MacOS chain.
+    Subject: {CN: "Trusted MacOS Kernel Test Server"}
+    Issuer: "macos-trusted-ca.pem"
+    pkcs12:
+      passphrase: "qwerty"
+      name: "macos-trusted-server.pfx"
+    extensions:
+      extendedKeyUsage: [serverAuth]
+      subjectAltName:
+        DNS: localhost
+        IP: 127.0.0.1