From 57cdbc0218860258f90436d4bb7d5c9a632a2b37 Mon Sep 17 00:00:00 2001 From: Janis Dombrovskis Date: Tue, 11 Aug 2026 01:46:28 +0300 Subject: [PATCH 01/39] Add SSH certificate authentication for targets, issued by Vault MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Warpgate authenticates to an SSH target with a short-lived OpenSSH user certificate signed on demand by HashiCorp Vault, instead of a private key it stores. The ephemeral keypair is generated per connection and never persisted, so a compromise of the Warpgate host yields nothing a target would accept. Targets trust the CA through TrustedUserCAKeys and need no authorized_keys. The certificate's key ID carries the Warpgate username and session UUID, so the target's own sshd log attributes a proxied session to a person rather than to the gateway. VaultAuth offers workload identity only — kubernetes, AppRole, AWS, Azure and GCP. Each reads its credential from a file or a metadata service, never from the config: a static Vault password would merely relocate the long-lived secret this feature exists to remove. Full compatibility with OpenBao is supported. Verified end to end against real infrastructure — AWS STS, a GCE instance, an Azure VM and a k3d cluster. tests/test_ssh_target_cert_auth.py runs against a stub issuer and needs neither Vault nor a cluster. Discussion: https://github.com/warp-tech/warpgate/issues/26 Special thanks to @theredspoon for the detailed test, OpenBao evaluation, and security recommendations. --- .github/workflows/docker.yml | 3 +- Cargo.lock | 23 + Cargo.toml | 1 + config-schema.json | 174 +++ tests/conftest.py | 9 +- tests/stub_vault.py | 408 +++++++ tests/test_ssh_target_cert_auth.py | 1051 +++++++++++++++++ warpgate-aws/src/error.rs | 18 + warpgate-aws/src/lib.rs | 2 + warpgate-aws/src/sts_identity.rs | 119 ++ warpgate-common/src/config/defaults.rs | 25 + warpgate-common/src/config/mod.rs | 101 +- warpgate-common/src/config/target.rs | 11 + warpgate-common/src/encryption.rs | 7 +- warpgate-core/Cargo.toml | 1 + warpgate-core/src/services.rs | 12 + warpgate-protocol-http/src/api/info.rs | 4 + warpgate-protocol-ssh/Cargo.toml | 1 + warpgate-protocol-ssh/src/client/mod.rs | 132 ++- warpgate-protocol-ssh/src/server/session.rs | 6 +- warpgate-vault/Cargo.toml | 27 + warpgate-vault/README.md | 343 ++++++ warpgate-vault/src/client.rs | 1003 ++++++++++++++++ warpgate-vault/src/error.rs | 77 ++ warpgate-vault/src/lib.rs | 6 + warpgate-vault/src/metadata.rs | 94 ++ .../admin/config/targets/ssh/Options.svelte | 12 + .../src/admin/lib/openapi-schema.json | 38 +- .../src/gateway/lib/openapi-schema.json | 7 +- warpgate/src/logging.rs | 7 +- 30 files changed, 3710 insertions(+), 12 deletions(-) create mode 100644 tests/stub_vault.py create mode 100644 tests/test_ssh_target_cert_auth.py create mode 100644 warpgate-aws/src/sts_identity.rs create mode 100644 warpgate-vault/Cargo.toml create mode 100644 warpgate-vault/README.md create mode 100644 warpgate-vault/src/client.rs create mode 100644 warpgate-vault/src/error.rs create mode 100644 warpgate-vault/src/lib.rs create mode 100644 warpgate-vault/src/metadata.rs diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index f1b10068e..df8c28888 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,7 +11,8 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME: warp-tech/warpgate + IMAGE_NAME: ${{ github.repository }} + concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/Cargo.lock b/Cargo.lock index e2c0c5c5d..44c0691ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9253,6 +9253,7 @@ dependencies = [ "warpgate-ldap", "warpgate-sso", "warpgate-tls", + "warpgate-vault", "webpki", "zune-jpeg", ] @@ -9550,6 +9551,7 @@ dependencies = [ "warpgate-core", "warpgate-db-entities", "warpgate-tls", + "warpgate-vault", "zeroize", ] @@ -9625,6 +9627,27 @@ dependencies = [ "x509-parser", ] +[[package]] +name = "warpgate-vault" +version = "0.27.4" +dependencies = [ + "data-encoding", + "rcgen", + "reqwest 0.13.4", + "rustls 0.23.43", + "rustls-pki-types", + "serde", + "serde_json", + "thiserror 2.0.19", + "tokio", + "tokio-rustls 0.26.4", + "tracing", + "url", + "warpgate-aws", + "warpgate-common", + "zeroize", +] + [[package]] name = "warpgate-web" version = "0.28.0-beta.1" diff --git a/Cargo.toml b/Cargo.toml index 247329f75..803aec6ab 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "warpgate-protocol-vnc", "warpgate-desktop-ui", "warpgate-sso", + "warpgate-vault", "warpgate-web", "warpgate-web-clients-common", "warpgate-web-desktop", diff --git a/config-schema.json b/config-schema.json index fbe685af4..37537f3cc 100644 --- a/config-schema.json +++ b/config-schema.json @@ -119,6 +119,17 @@ "$ref": "#/$defs/SsoProviderConfig" } }, + "vault": { + "description": "Absent unless the deployment issues target credentials from Vault.", + "anyOf": [ + { + "$ref": "#/$defs/VaultConfig" + }, + { + "type": "null" + } + ] + }, "vnc": { "$ref": "#/$defs/VncConfig", "default": { @@ -804,6 +815,169 @@ "host_header" ] }, + "VaultAuth": { + "description": "How Warpgate proves its own identity to Vault.\n\nBoth methods read their credential from a file rather than from the config,\ndeliberately: the point of issuing target credentials on demand is that\nnothing long-lived sits on the Warpgate host, and a value pasted into the\nconfig would put it straight back. A Kubernetes service account token is\nmounted and rotated by the kubelet; an AppRole secret ID is short-lived and\nmeant to be delivered response-wrapped by whatever provisions the host.", + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "kubernetes" + }, + "role": { + "type": "string" + }, + "token_path": { + "type": "string", + "default": "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + }, + "required": [ + "kind", + "role" + ] + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "app_role" + }, + "role_id": { + "type": "string" + }, + "secret_id_path": { + "type": "string" + } + }, + "required": [ + "kind", + "role_id", + "secret_id_path" + ] + }, + { + "description": "Signs an `sts:GetCallerIdentity` call with the default credential chain —\non EC2 that is the instance role. Access keys in the environment work but\nput back the long-lived secret this exists to avoid.", + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "aws" + }, + "region": { + "description": "Signs against a regional STS endpoint instead of the global one. Set\nthis only when Vault is configured with a matching `sts_endpoint`:\nVault replays the request globally by default, and a region-scoped\nsignature is rejected there.", + "type": [ + "string", + "null" + ], + "default": null + }, + "role": { + "type": [ + "string", + "null" + ], + "default": null + }, + "server_id": { + "description": "Bound into the signature so a captured request cannot be replayed\nagainst a different Vault. Must match the server's `iam_server_id_header_value`.", + "type": [ + "string", + "null" + ], + "default": null + } + }, + "required": [ + "kind" + ] + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "azure" + }, + "metadata_address": { + "type": "string", + "default": "http://169.254.169.254" + }, + "resource": { + "type": "string", + "default": "https://management.azure.com/" + }, + "role": { + "type": "string" + } + }, + "required": [ + "kind", + "role" + ] + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "gcp" + }, + "metadata_address": { + "type": "string", + "default": "http://metadata.google.internal" + }, + "role": { + "type": "string" + } + }, + "required": [ + "kind", + "role" + ] + } + ] + }, + "VaultConfig": { + "type": "object", + "properties": { + "address": { + "description": "Base URL of the Vault server, e.g. `https://vault.internal:8200`.", + "type": "string" + }, + "auth": { + "$ref": "#/$defs/VaultAuth" + }, + "certificate_ttl": { + "description": "Lifetime asked for when signing. Vault clamps this to the role's\n`max_ttl`, so it can only shorten what the role already allows — set it\nto hold the window down without editing Vault. Left unset, the role's own\nTTL decides.", + "type": [ + "string", + "null" + ], + "default": null + }, + "default_role": { + "description": "Signing role used by targets that don't name one of their own.", + "type": "string" + }, + "mount": { + "description": "Mount point of the SSH secrets engine in signed-certificates mode.", + "type": "string", + "default": "ssh-client-signer" + }, + "timeout": { + "type": "string", + "default": "10s" + } + }, + "required": [ + "address", + "default_role", + "auth" + ] + }, "VncConfig": { "type": "object", "properties": { diff --git a/tests/conftest.py b/tests/conftest.py index d0f3b657c..f9987977d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -144,18 +144,24 @@ def stop(self): pass p.kill() - def start_ssh_server(self, trusted_keys=[], extra_config=""): + def start_ssh_server(self, trusted_keys=[], extra_config="", trusted_ca=[]): port = alloc_port() data_dir = self.ctx.tmpdir / f"sshd-{uuid.uuid4()}" data_dir.mkdir(parents=True) authorized_keys_path = data_dir / "authorized_keys" authorized_keys_path.write_text("\n".join(trusted_keys)) + # Always written, even when empty: an unreadable TrustedUserCAKeys file + # makes sshd reject every connection, which is indistinguishable from the + # certificate being wrong. + trusted_ca_path = data_dir / "trusted_ca" + trusted_ca_path.write_text("\n".join(trusted_ca)) config_path = data_dir / "sshd_config" config_path.write_text( dedent( f"""\ Port 22 AuthorizedKeysFile {authorized_keys_path} + TrustedUserCAKeys {trusted_ca_path} AllowAgentForwarding yes AllowTcpForwarding yes GatewayPorts yes @@ -173,6 +179,7 @@ def start_ssh_server(self, trusted_keys=[], extra_config=""): ) data_dir.chmod(0o700) authorized_keys_path.chmod(0o600) + trusted_ca_path.chmod(0o600) config_path.chmod(0o600) self.start( diff --git a/tests/stub_vault.py b/tests/stub_vault.py new file mode 100644 index 000000000..0d2008d8f --- /dev/null +++ b/tests/stub_vault.py @@ -0,0 +1,408 @@ +"""A stand-in for Vault's SSH secrets engine. + +Signs with a throwaway CA through ``ssh-keygen``, so the certificate tests need +neither a Vault server nor a cluster. Every knob here exists to reproduce a +failure Warpgate has to survive, and every recorded request exists so a test can +assert on what Warpgate actually asked for. +""" + +import json +import subprocess +import tempfile +import threading +import time +from base64 import b64decode, urlsafe_b64decode, urlsafe_b64encode +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +MOUNT = "ssh-client-signer" + + +def jwt(claims: dict) -> str: + """A JWT-shaped token. Unsigned — the stub checks shape and claims, which is + what catches a payload built wrong, not cryptographic validity.""" + + def segment(raw: bytes) -> str: + return urlsafe_b64encode(raw).rstrip(b"=").decode() + + return ".".join( + [ + segment(json.dumps({"alg": "RS256", "typ": "JWT"}).encode()), + segment(json.dumps(claims).encode()), + segment(b"stub-signature"), + ] + ) + + +def jwt_claims(token: str) -> dict: + payload = token.split(".")[1] + return json.loads(urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + + +def is_jwt(token) -> bool: + parts = (token or "").split(".") + if len(parts) != 3 or not all(parts): + return False + try: + jwt_claims(token) + except Exception: + return False + return True + + +SERVICE_ACCOUNT_JWT = jwt( + {"iss": "kubernetes/serviceaccount", "sub": "system:serviceaccount:warpgate:warpgate"} +) + + +def reject_login(method, body): + """Checks what the real auth method checks, in the same order it does. + + A stub that accepts anything turns every login test into an assertion + about the stub instead of about the payload Warpgate built. + """ + if method == "kubernetes": + if not body.get("role"): + return "missing role" + if not is_jwt(body.get("jwt")): + return "jwt is not a JWT" + return None + + if method == "approle": + if not body.get("role_id") or not body.get("secret_id"): + return "missing role_id or secret_id" + return None + + if method == "aws": + return reject_aws_login(body) + + if method == "azure": + for field in ("subscription_id", "resource_group_name", "vm_name"): + if not body.get(field): + return f"missing {field}" + if "vmss_name" in body and not body["vmss_name"]: + return "vmss_name present but empty" + if not is_jwt(body.get("jwt")): + return "jwt is not a JWT" + return None + + if method == "gcp": + role = body.get("role") + if not role: + return "missing role" + if not is_jwt(body.get("jwt")): + return "jwt is not a JWT" + # Vault rejects a token minted for a different audience, which is + # what stops a token issued for one role being replayed at another. + if jwt_claims(body["jwt"]).get("aud") != f"vault/{role}": + return "jwt audience is not bound to the role" + return None + + return f"unknown auth method {method}" + +def reject_aws_login(body): + """Vault replays this request against STS, so it has to be a signed + GetCallerIdentity call and not merely four non-empty fields.""" + if body.get("iam_http_request_method") != "POST": + return "iam_http_request_method must be POST" + try: + url = b64decode(body.get("iam_request_url", "")).decode() + signed_body = b64decode(body.get("iam_request_body", "")).decode() + headers = json.loads(b64decode(body.get("iam_request_headers", ""))) + except Exception as e: + return f"undecodable IAM payload: {e}" + + host = urlsplit(url).hostname or "" + if host != "sts.amazonaws.com" and not ( + host.startswith("sts.") and host.endswith(".amazonaws.com") + ): + return f"not an STS endpoint: {host}" + if parse_qs(signed_body).get("Action") != ["GetCallerIdentity"]: + return f"not a GetCallerIdentity call: {signed_body}" + + authorization = next( + (v for k, v in headers.items() if k.lower() == "authorization"), "" + ) + if not authorization.startswith("AWS4-HMAC-SHA256"): + return "missing SigV4 Authorization header" + return None + + +class Recorder: + """Records every request it receives and nothing else. Used to prove that a + redirect from the issuer never reaches its target.""" + + def __init__(self): + self.requests = [] + handler = self + + class _RecordingHandler(BaseHTTPRequestHandler): + def do_GET(self): + self._record() + + def do_POST(self): + self._record() + + def _record(self): + handler.requests.append((self.path, dict(self.headers))) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), _RecordingHandler) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def start(self): + self._thread.start() + + def stop(self): + self._server.shutdown() + self._thread.join(timeout=5) + self._server.server_close() + + @property + def url(self): + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + +class _Handler(BaseHTTPRequestHandler): + def do_GET(self): + """The cloud metadata services Warpgate reads its identity from.""" + stub = self.server.stub + path = self.path.split("?")[0] + stub.requests.append(self.path) + stub.metadata_requests.append(self.path) + + query = parse_qs(urlsplit(self.path).query) + + if path == "/metadata/identity/oauth2/token": + resource = query.get("resource", [""])[0] + self._reply(200, {"access_token": jwt({"aud": resource, "oid": "oid-1"})}) + elif path == "/metadata/instance/compute": + self._reply(200, { + "subscriptionId": "sub-1", + "resourceGroupName": "rg-1", + "name": "vm-1", + "vmScaleSetName": "", + }) + elif path.endswith("/service-accounts/default/identity"): + audience = query.get("audience", [""])[0] + self._send_text(200, jwt({"aud": audience, "sub": "gce-instance"}) + "\n") + else: + self._reply(404, {"errors": [f"no handler for {path}"]}) + + def _send_text(self, status, body): + encoded = body.encode() + self.send_response(status) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def do_POST(self): + stub = self.server.stub + stub.requests.append(self.path) + length = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(length) or b"{}") + + if self.path == "/v1/sys/wrapping/unwrap": + token = self.headers.get("X-Vault-Token", "") + if not token: + self._reply(400, {"errors": ["missing wrapping token"]}) + return + self._reply(200, {"data": {"secret_id": "unwrapped-secret-id"}}) + return + + if self.path.startswith("/v1/auth/") and self.path.endswith("/login"): + method = self.path.split("/")[3] + rejection = reject_login(method, body) + if rejection: + self._reply(400, {"errors": [rejection]}) + return + + stub.logins.append({"method": method, **body}) + + # Vault is not instantaneous, and how many sessions can be inside a + # single login at once is the thing some tests are about. + time.sleep(stub.login_delay) + + stub.valid_token = f"stub-token-{len(stub.logins)}" + self._reply( + 200, + { + "auth": { + "client_token": stub.valid_token, + "lease_duration": stub.lease_duration, + } + }, + ) + return + + if self.path.startswith(f"/v1/{MOUNT}/sign/"): + self._sign(stub, self.path.rsplit("/", 1)[-1], body) + return + + self._reply(404, {"errors": [f"no handler for {self.path}"]}) + + + def _sign(self, stub, role, body): + presented = self.headers.get("X-Vault-Token") + stub.signs.append({"role": role, "token": presented, **body}) + + if stub.sign_redirect_to is not None: + self.send_response(307) + self.send_header("Location", stub.sign_redirect_to) + self.send_header("Content-Length", "0") + self.end_headers() + return + + if stub.sign_error_body is not None: + self._send_raw(500, stub.sign_error_body) + return + + if stub.sign_status is not None: + self._reply(stub.sign_status, {"errors": ["stub refuses to sign"]}) + return + + # Mirrors Vault rejecting a token that was revoked, or that predates a + # restart, before its lease was due to expire. Only checked once a test + # asks for it, so a token cached by an earlier test cannot leak into an + # unrelated one. + if not stub.accept_any_token and presented != stub.valid_token: + self._reply(403, {"errors": ["permission denied", "invalid token"]}) + return + + if stub.sign_data is not None: + self._reply(200, {"data": stub.sign_data}) + return + + if stub.signed_key is not None: + self._reply(200, {"data": {"signed_key": stub.signed_key}}) + return + + certificate = stub.issue( + public_key=stub.sign_public_key or body["public_key"], + principals=( + stub.principals if stub.principals is not None else body["valid_principals"] + ), + key_id=body.get("key_id", ""), + ) + self._reply(200, {"data": {"signed_key": certificate}}) + + def _reply(self, status, payload): + self._send_raw(status, json.dumps(payload).encode(), "application/json") + + def _send_raw(self, status, encoded: bytes, content_type="application/json"): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + try: + self.wfile.write(encoded) + except (BrokenPipeError, ConnectionResetError): + # Warpgate hangs up on a body larger than it will accept, which is + # exactly what one of the tests here is checking for. + pass + + def log_message(self, *args): + pass + + +class StubVault: + def __init__(self, directory: Path): + directory.mkdir(parents=True, exist_ok=True) + self.directory = directory + self.ca_key = directory / "ca" + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-f", str(self.ca_key), "-N", ""], + check=True, + ) + self._server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + self._server.stub = self + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self.logins = [] + self.signs = [] + self.requests = [] + self.metadata_requests = [] + self.valid_token = None + self.reset() + + def reset(self): + """Restores the defaults a happy-path test expects.""" + self.lease_duration = 3600 + self.login_delay = 0 + self.accept_any_token = True + self.sign_status = None + self.sign_error_body = None + self.sign_redirect_to = None + self.signed_key = None + self.sign_data = None + self.sign_public_key = None + self.principals = None + self.validity = "-30s:+2m" + self.cert_type = "user" + self.sign_options = [] + self.logins.clear() + self.signs.clear() + self.requests.clear() + self.metadata_requests.clear() + + def start(self): + self._thread.start() + + def stop(self): + self._server.shutdown() + self._thread.join(timeout=5) + self._server.server_close() + + @property + def url(self): + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + @property + def ca_public_key(self) -> str: + return Path(f"{self.ca_key}.pub").read_text().strip() + + def issue(self, public_key: str, principals: str, key_id: str) -> str: + with tempfile.TemporaryDirectory() as directory: + key = Path(directory) / "key.pub" + key.write_text(public_key) + # ssh-keygen decides itself which of `sign_options` are critical + # options and which are extensions, exactly as a Vault role's + # default_critical_options and default_extensions end up doing. + options = [arg for option in self.sign_options for arg in ("-O", option)] + subprocess.run( + [ + "ssh-keygen", "-q", + "-s", str(self.ca_key), + "-I", key_id, + "-n", principals, + "-V", self.validity, + *(["-h"] if self.cert_type == "host" else []), + *options, + str(key), + ], + check=True, + ) + return (Path(directory) / "key-cert.pub").read_text() + + def unrelated_public_key(self) -> str: + """A public key Warpgate does not hold the private half of.""" + path = self.directory / "unrelated" + if not path.exists(): + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-f", str(path), "-N", ""], + check=True, + ) + return Path(f"{path}.pub").read_text().strip() + + def invalidate_token(self): + """Makes the cached token stop working, as a Vault restart would.""" + self.accept_any_token = False + self.valid_token = "rotated-away" diff --git a/tests/test_ssh_target_cert_auth.py b/tests/test_ssh_target_cert_auth.py new file mode 100644 index 000000000..93bc99462 --- /dev/null +++ b/tests/test_ssh_target_cert_auth.py @@ -0,0 +1,1051 @@ +"""Target authentication with short-lived certificates issued by Vault. + +Beyond the happy path, the cases here are drawn from two places: the failures +this feature hit while being built, and the classes of bug Warpgate has actually +shipped before — a credential accepted without proof of possession +(GHSA-3cjp-w4cp-m9c8), a new code path skipping a control every other path +applies (GHSA-qmr2-wp96-h9ff), and an identity taken from the wrong stage of +authentication (GHSA-c94j-vqr5-3mxr). +""" + +import json +from pathlib import Path +from uuid import uuid4 + +import psutil +import pytest + +from .api_client import admin_client, sdk +from .conftest import ProcessManager, WarpgateProcess +from .stub_vault import ( + SERVICE_ACCOUNT_JWT, + Recorder, + StubVault, + jwt_claims, + reject_login, +) +from .util import wait_port + +USER_PUBLIC_KEY_PATH = Path("ssh-keys/id_ed25519.pub") +USER_PRIVATE_KEY_PATH = "ssh-keys/id_ed25519" + + +@pytest.fixture(scope="module") +def stub_vault(ctx): + stub = StubVault(ctx.tmpdir / f"stub-vault-{uuid4()}") + stub.start() + yield stub + stub.stop() + + +@pytest.fixture(scope="module") +def cert_wg(processes: ProcessManager, ctx, stub_vault: StubVault): + """Warpgate wired to the stub issuer, with its log kept for assertions.""" + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + log_path = ctx.tmpdir / f"cert-wg-{uuid4()}.log" + with log_path.open("w") as log: + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + }, + stdout=log, + stderr=log, + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + wg.log_path = log_path + yield wg + + +@pytest.fixture(scope="module") +def cert_ssh_port(processes: ProcessManager, stub_vault: StubVault): + """A target that trusts the stub CA and has no authorized_keys at all.""" + port = processes.start_ssh_server(trusted_ca=[stub_vault.ca_public_key]) + wait_port(port) + return port + + +@pytest.fixture(autouse=True) +def reset_stub(stub_vault: StubVault): + stub_vault.reset() + yield + stub_vault.reset() + + +def make_user_and_target(api, ssh_port, *, role=None, username="root", assign=True): + wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user(sdk.CreateUserRequest(username=f"user-{uuid4()}")) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, wg_role.id) + + target = api.create_target( + sdk.TargetDataRequest( + name=f"cert-{uuid4()}", + options=sdk.TargetOptions( + sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host="localhost", + port=ssh_port, + username=username, + auth=sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetCertificateAuth( + kind="Certificate", role=role + ) + ), + ) + ), + ) + ) + if assign: + api.add_target_role(target.id, wg_role.id) + return user, target + + +def start(processes: ProcessManager, wg: WarpgateProcess, user, target, *extra): + return processes.start_ssh_client( + f"{user.username}:{target.name}@localhost", + "-p", + str(wg.ssh_port), + "-o", + f"IdentityFile={USER_PRIVATE_KEY_PATH}", + "-o", + "PreferredAuthentications=publickey", + *extra, + "ls", + "/bin/sh", + ) + + +def connect(processes: ProcessManager, wg: WarpgateProcess, user, target, timeout, *extra): + client = start(processes, wg, user, target, *extra) + stdout = client.communicate(timeout=timeout)[0] + return client.returncode, stdout + + +def log_since(wg: WarpgateProcess, offset: int) -> str: + """Only what this test wrote. The gateway is shared by the whole module, so + searching the whole log would find another test's line just as happily.""" + return Path(wg.log_path).read_text(errors="replace")[offset:] + + +@pytest.fixture +def api(cert_wg: WarpgateProcess): + with admin_client(f"https://localhost:{cert_wg.http_port}") as client: + yield client + + +class TestHappyPath: + def test_connects_with_an_issued_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert code == 0 + assert stdout == b"/bin/sh\n" + assert len(stub_vault.signs) == 1 + assert stub_vault.signs[0]["valid_principals"] == "root" + assert stub_vault.signs[0]["cert_type"] == "user" + + def test_uses_the_default_role_unless_the_target_names_one( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + assert stub_vault.signs[-1]["role"] == "warpgate" + + user, target = make_user_and_target(api, cert_ssh_port, role="privileged") + connect(processes, cert_wg, user, target, timeout) + assert stub_vault.signs[-1]["role"] == "privileged" + + def test_each_session_gets_a_fresh_key_and_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + for _ in range(3): + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + offered = [sign["public_key"] for sign in stub_vault.signs] + assert len(offered) == 3 + assert len(set(offered)) == 3, "an ephemeral key was reused between sessions" + + def test_the_token_is_reused_across_sessions( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + for _ in range(3): + connect(processes, cert_wg, user, target, timeout) + + # The first session may find a token cached by an earlier test, so the + # assertion is that three sessions do not each cause a login. + assert len(stub_vault.logins) <= 1 + + def test_no_ttl_is_requested_unless_one_is_configured( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Vault reads an absent `ttl` as the role's default. Sending a zero + instead would ask for the shortest certificate Vault will issue.""" + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + assert "ttl" not in stub_vault.signs[-1] + + def test_a_configured_ttl_is_asked_for( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """The TTL can be held down from Warpgate's side without editing the + Vault role — Vault clamps it to the role's `max_ttl` regardless.""" + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "certificate_ttl": "90s", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + assert stub_vault.signs[-1]["ttl"] == "90s" + + +class TestIdentity: + """The certificate must name the user Warpgate actually authenticated.""" + + def test_key_id_carries_the_authenticated_user_and_session( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + key_id = stub_vault.signs[-1]["key_id"] + prefix, username, session = key_id.split(":") + assert prefix == "warpgate" + assert username == user.username + assert session + + def test_the_target_username_is_not_the_warpgate_username( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """`valid_principals` bounds access on the target and must never be + taken from the identity the client chose to log into Warpgate with.""" + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + assert stub_vault.signs[-1]["valid_principals"] == "root" + assert user.username not in stub_vault.signs[-1]["valid_principals"] + + def test_a_comma_in_the_target_username_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Vault reads `valid_principals` as a comma-separated list, so a comma + in the target's username would ask for a certificate good for accounts + nobody named. No request may leave at all.""" + user, target = make_user_and_target(api, cert_ssh_port, username="deploy,root") + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + + assert stub_vault.signs == [] + + def test_a_newline_in_the_target_username_never_reaches_vault( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The principal is written into the certificate and the certificate is + written into the target's sshd log, so a newline in a target's username + is a way to compose log lines on the target. Nothing may be sent at all + — not even the login that would precede it.""" + user, target = make_user_and_target(api, cert_ssh_port, username="root\nnobody") + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + + assert stub_vault.requests == [] + + def test_a_ticket_session_is_named_after_its_user_and_not_its_secret( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The key ID must name the user Warpgate authenticated, however it + authenticated them. A ticket session logs in as `ticket-`, so a + key ID taken from the login name rather than from the authentication + result would name nobody and copy the ticket secret into the target's + sshd log — the shape of GHSA-c94j-vqr5-3mxr.""" + user, target = make_user_and_target(api, cert_ssh_port) + secret = api.create_ticket( + sdk.CreateTicketRequest(target_name=target.name, username=user.username) + ).secret + + client = processes.start_ssh_client( + f"ticket-{secret}@localhost", + "-p", + str(cert_wg.ssh_port), + "-o", + "PreferredAuthentications=password", + "-i", + "/dev/null", + "ls", + "/bin/sh", + password="irrelevant", + ) + assert client.communicate(timeout=timeout)[0] == b"/bin/sh\n" + + key_id = stub_vault.signs[-1]["key_id"] + assert key_id.startswith(f"warpgate:{user.username}:") + assert secret not in key_id + + +class TestRejections: + def test_target_that_does_not_trust_the_ca( + self, processes, cert_wg, stub_vault, api, timeout + ): + port = processes.start_ssh_server() + wait_port(port) + + user, target = make_user_and_target(api, port) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert code != 0 + assert stdout == b"" + assert len(stub_vault.signs) == 1, "the target rejected it, not Warpgate" + + def test_expired_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.validity = "-2h:-1h" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + + def test_certificate_that_is_not_yet_valid( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A target whose clock lags far enough behind Warpgate's sees this.""" + stub_vault.validity = "+1h:+2h" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + + def test_certificate_for_a_different_principal( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.principals = "nobody" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + + def test_certificate_issued_for_a_key_warpgate_does_not_hold( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A certificate is a public credential. Signing someone else's key must + not authenticate anyone — the class of bug behind GHSA-3cjp-w4cp-m9c8, + where an SSH key offer was accepted without a signature.""" + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.sign_public_key = stub_vault.unrelated_public_key() + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + # The target would refuse it too, but then the log would say only that + # the target said no. Warpgate knows which key it generated. + assert "signed a key other than" in log_since(cert_wg, offset) + + def test_a_host_certificate_is_not_offered_to_the_target( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A host certificate can never authenticate a user, so an issuer that + returns one is misconfigured or lying. Either way the reason belongs in + Warpgate's log rather than arriving as an unexplained rejection.""" + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.cert_type = "host" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + assert "host certificate" in log_since(cert_wg, offset) + + def test_malformed_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.signed_key = "this is not a certificate" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + + @pytest.mark.parametrize( + "data", + [ + pytest.param({}, id="no-signed-key"), + pytest.param({"signed_key": ""}, id="empty-signed-key"), + pytest.param({"signed_key": 42}, id="signed-key-is-not-a-string"), + ], + ) + def test_a_success_that_carries_no_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout, data + ): + """`200 OK` is not a certificate. Each of these parses as JSON and none + of them is a credential, so the session has to end rather than continue + with whatever an absent field defaults to.""" + stub_vault.sign_data = data + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + + def test_a_forced_command_from_the_issuer_is_reported( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The target's sshd enforces critical options, so an issuer that + attaches `force-command` decides what the session runs — the user's own + command never executes, and the recording shows the output of something + they did not type. Warpgate asks for no critical options and cannot stop + the target honouring one, so the least it can do is say one arrived.""" + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.sign_options = ["force-command=echo chosen-by-the-issuer"] + user, target = make_user_and_target(api, cert_ssh_port) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + # Not an assumption: this is the mechanism the test exists for. + assert (code, stdout) == (0, b"chosen-by-the-issuer\n") + + log = log_since(cert_wg, offset) + assert "critical options" in log + assert "force-command" in log + + +class TestIssuerFailures: + def test_issuer_refuses_to_sign( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.sign_status = 403 + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + + def test_a_persistent_denial_is_not_retried_forever( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """One re-login distinguishes a stale token from a policy denial. A + second failure has to be final, or a denied target becomes a request + amplifier against Vault.""" + stub_vault.sign_status = 403 + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + assert len(stub_vault.signs) == 2 + + def test_a_stale_token_is_refreshed_once( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Vault can reject a token long before its lease runs out — after a + restart, or a revocation. Warpgate must recover within the session + rather than failing until the lease expires.""" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + stub_vault.invalidate_token() + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + assert len(stub_vault.signs) == 3, "expected one rejected and one retried sign" + + def test_a_redirect_from_the_issuer_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Following a redirect would hand `X-Vault-Token` to whatever host it + names — reqwest only knows to strip `Authorization`. The session must + fail instead, and the redirect target must never be contacted.""" + recorder = Recorder() + recorder.start() + try: + stub_vault.sign_redirect_to = f"{recorder.url}/v1/steal" + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert stub_vault.signs, "the signing request was never made" + assert recorder.requests == [], "the redirect was followed" + finally: + recorder.stop() + + def test_an_oversized_error_body_is_not_relayed_to_the_client( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """An endpoint answering on the Vault address can return any body it + likes. It must neither be buffered whole nor shown to the user.""" + marker = "SENSITIVE-INTERNAL-DETAIL" + stub_vault.sign_error_body = (marker + "x" * 64).encode() * 200_000 + + user, target = make_user_and_target(api, cert_ssh_port) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert code != 0 + assert marker not in stdout.decode(errors="replace") + + def test_an_oversized_signing_response_is_not_buffered_whole( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The failed-response path was bounded; the successful one is the same + body with a different status code on it, and it is parsed once per + session. Left unbounded it is memory the issuer gets to allocate inside + Warpgate, as often as sessions are started.""" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + gateway = psutil.Process(cert_wg.process.pid) + before = gateway.memory_info().rss + stub_vault.signed_key = "ssh-ed25519-cert-v01@openssh.com " + "A" * 100_000_000 + assert connect(processes, cert_wg, user, target, timeout * 3)[0] != 0 + growth = gateway.memory_info().rss - before + + assert growth < 50_000_000, f"the body was buffered whole ({growth / 1e6:.0f} MB)" + + # And the gateway is still there, with the memory it started with. + stub_vault.signed_key = None + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + def test_a_revoked_token_costs_one_login_no_matter_how_many_sessions_find_it( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A Vault restart is discovered by every session in flight at the same + moment. If each answered by logging in, ordinary traffic would meet the + restart with a login per session — and each of those with a credential + read off disk.""" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + stub_vault.invalidate_token() + # Wide enough that the sessions are genuinely inside the same login, + # rather than politely arriving one after another. + stub_vault.login_delay = 1 + logins = len(stub_vault.logins) + + clients = [start(processes, cert_wg, user, target) for _ in range(6)] + for client in clients: + client.communicate(timeout=timeout * 3) + + assert [client.returncode for client in clients] == [0] * 6 + assert len(stub_vault.logins) - logins == 1 + + def test_the_client_is_never_shown_the_issuers_own_words( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """What the user sees comes from a fixed list; the issuer's body is + written for operators and names mounts, policies and hosts. This asks + for a PTY on purpose — without one the message has nowhere to go and the + check would pass without anything being shown at all.""" + stub_vault.sign_error_body = b"1 error occurred: permission denied by policy ssh-signer-7" + user, target = make_user_and_target(api, cert_ssh_port) + code, stdout = connect(processes, cert_wg, user, target, timeout, "-tt") + + assert code != 0 + shown = stdout.decode(errors="replace") + assert "Vault service error" in shown, "the failure never reached the user" + assert "ssh-signer-7" not in shown + assert "policy" not in shown + + def test_an_error_body_split_mid_character_does_not_kill_the_session( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Truncating at a fixed byte count lands inside a multi-byte character + for some bodies; slicing a Rust `String` there panics.""" + stub_vault.sign_error_body = b"a" * 255 + "é".encode() + b"b" * 64 + + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + + # The gateway has to still be there afterwards: a panic in the signing + # path would take the session task down and leave the next login hanging. + stub_vault.sign_error_body = None + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + def test_issuer_unreachable_fails_promptly( + self, processes: ProcessManager, ctx, cert_ssh_port, timeout + ): + """An unreachable Vault must fail the session rather than stall it, so + an issuer outage looks like an auth failure and not a hang.""" + dead = StubVault(ctx.tmpdir / f"dead-vault-{uuid4()}") + dead.start() + address = dead.url + dead.stop() + + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": address, + "default_role": "warpgate", + "timeout": "5s", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + # The assertion is the absence of a timeout: communicate() raises if the + # session is still open when the test's own deadline passes. + code, _ = connect(processes, wg, user, target, timeout) + assert code != 0 + + +class TestAuthMethods: + def test_kubernetes_sends_the_service_account_token( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + # An earlier test may have left a usable token cached, so force a login. + stub_vault.invalidate_token() + connect(processes, cert_wg, user, target, timeout) + + login = stub_vault.logins[-1] + assert login["method"] == "kubernetes" + assert login["role"] == "warpgate" + assert login["jwt"] == SERVICE_ACCOUNT_JWT + + def test_approle_sends_the_secret_id_from_its_file( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """The secret ID is read from disk rather than the config so it can be + rotated underneath a running Warpgate.""" + secret_id_path = ctx.tmpdir / f"secret-id-{uuid4()}" + secret_id_path.write_text("stub-secret-id\n") + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "app_role", + "role_id": "stub-role-id", + "secret_id_path": str(secret_id_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + login = stub_vault.logins[-1] + assert login["method"] == "approle" + assert login["role_id"] == "stub-role-id" + assert login["secret_id"] == "stub-secret-id" + + +class TestCloudAuthMethods: + """The cloud methods take their credential from a metadata service, so + nothing durable is written to the host at all. Only the request Warpgate + builds is under test here; the metadata services themselves need a real VM.""" + + def _wg_with_auth(self, processes, stub_vault, auth, env=None): + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": auth, + } + }, + env=env, + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + return wg + + def test_azure_sends_imds_token_and_vm_coordinates( + self, processes: ProcessManager, stub_vault, cert_ssh_port, timeout + ): + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "azure", + "role": "warpgate", + "metadata_address": stub_vault.url, + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + login = stub_vault.logins[-1] + assert login["method"] == "azure" + assert jwt_claims(login["jwt"])["aud"] == "https://management.azure.com/" + assert login["subscription_id"] == "sub-1" + assert login["resource_group_name"] == "rg-1" + assert login["vm_name"] == "vm-1" + assert any("management.azure.com" in r for r in stub_vault.metadata_requests) + + def test_gcp_requests_a_token_bound_to_its_role( + self, processes: ProcessManager, stub_vault, cert_ssh_port, timeout + ): + """The audience ties the token to one Vault role, so a token minted for + another role cannot be presented here.""" + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "gcp", + "role": "warpgate", + "metadata_address": stub_vault.url, + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + login = stub_vault.logins[-1] + assert login["method"] == "gcp" + assert jwt_claims(login["jwt"])["aud"] == "vault/warpgate" + assert any( + "audience=vault%2Fwarpgate" in r for r in stub_vault.metadata_requests + ) + + def test_aws_signs_the_global_endpoint_by_default( + self, processes: ProcessManager, stub_vault, cert_ssh_port, timeout + ): + """Vault replays the request against the global STS endpoint, which only + accepts signatures scoped to us-east-1. Signing a regional endpoint by + default makes every login fail with SignatureDoesNotMatch.""" + from base64 import b64decode + + wg = self._wg_with_auth( + processes, + stub_vault, + {"kind": "aws", "role": "warpgate"}, + env={ + "AWS_ACCESS_KEY_ID": "ASIA0000000000000000", + "AWS_SECRET_ACCESS_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "AWS_SESSION_TOKEN": "AQoDYXdzEJr1KEXAMPLEtoken", + }, + + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + connect(processes, wg, user, target, timeout) + + login = stub_vault.logins[-1] + assert b64decode(login["iam_request_url"]) == b"https://sts.amazonaws.com/" + + headers = json.loads(b64decode(login["iam_request_headers"])) + authorization = next( + value for name, value in headers.items() if name.lower() == "authorization" + ) + assert "/us-east-1/sts/aws4_request" in authorization + + def test_aws_sends_a_signed_sts_request( + self, processes: ProcessManager, stub_vault, cert_ssh_port, timeout + ): + """Vault replays the signed request against STS, so the signature — not a + disclosed credential — is what proves identity.""" + from base64 import b64decode + + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "aws", + "role": "warpgate", + "region": "us-east-1", + "server_id": "vault.example.com", + }, + # Supplied here rather than inherited, so the test does not depend on + # whatever credentials the developer happens to have. The signature is + # verified for shape, never sent to AWS. + env={ + "AWS_ACCESS_KEY_ID": "ASIA0000000000000000", + "AWS_SECRET_ACCESS_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "AWS_SESSION_TOKEN": "AQoDYXdzEJr1KEXAMPLEtoken", + }, + + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + connect(processes, wg, user, target, timeout) + + login = stub_vault.logins[-1] + assert login["method"] == "aws" + assert login["iam_http_request_method"] == "POST" + assert b64decode(login["iam_request_url"]) == b"https://sts.us-east-1.amazonaws.com/" + assert b64decode(login["iam_request_body"]) == ( + b"Action=GetCallerIdentity&Version=2011-06-15" + ) + + headers = json.loads(b64decode(login["iam_request_headers"])) + headers = {name.lower(): value for name, value in headers.items()} + assert headers["x-vault-aws-iam-server-id"] == "vault.example.com" + assert "AWS4-HMAC-SHA256" in headers["authorization"] + assert "x-vault-aws-iam-server-id" in headers["authorization"], ( + "the server ID must be signed, not merely sent" + ) + + def test_approle_response_wrapping( + self, processes, cert_ssh_port, stub_vault, ctx, timeout + ): + secret_id_path = ctx.tmpdir / f"wrapping-token-{uuid4()}" + secret_id_path.write_text("unwrap:stub-wrapping-token") + + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "app_role", + "role_id": "role-1", + "secret_id_path": str(secret_id_path), + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + login = stub_vault.logins[-1] + assert login["method"] == "approle" + assert login["secret_id"] == "unwrapped-secret-id" + + + +class TestTheStubItself: + """The auth-method tests only mean something if the stub would have noticed a + wrong payload. These check the checker, without starting anything.""" + + def _aws_payload(self, **overrides): + from base64 import b64encode + + payload = { + "iam_http_request_method": "POST", + "iam_request_url": b64encode(b"https://sts.amazonaws.com/").decode(), + "iam_request_body": b64encode( + b"Action=GetCallerIdentity&Version=2011-06-15" + ).decode(), + "iam_request_headers": b64encode( + json.dumps({"authorization": "AWS4-HMAC-SHA256 Credential=..."}).encode() + ).decode(), + } + payload.update(overrides) + return payload + + def test_a_well_formed_aws_payload_is_accepted(self): + assert reject_login("aws", self._aws_payload()) is None + + @pytest.mark.parametrize( + "overrides", + [ + {"iam_http_request_method": "GET"}, + {"iam_request_url": "aHR0cHM6Ly9ldmlsLmV4YW1wbGUuY29tLw=="}, # not STS + {"iam_request_body": "QWN0aW9uPUFzc3VtZVJvbGU="}, # not GetCallerIdentity + {"iam_request_headers": "e30="}, # {} — unsigned + {"iam_request_url": "not base64 at all"}, + ], + ) + def test_a_mangled_aws_payload_is_rejected(self, overrides): + assert reject_login("aws", self._aws_payload(**overrides)) is not None + + def test_a_non_jwt_identity_token_is_rejected(self): + assert reject_login("kubernetes", {"role": "warpgate", "jwt": "a-token"}) + assert reject_login("kubernetes", {"role": "warpgate", "jwt": SERVICE_ACCOUNT_JWT}) is None + + def test_a_gcp_token_for_another_role_is_rejected(self): + from .stub_vault import jwt + + assert reject_login( + "gcp", {"role": "warpgate", "jwt": jwt({"aud": "vault/other-role"})} + ) + assert ( + reject_login("gcp", {"role": "warpgate", "jwt": jwt({"aud": "vault/warpgate"})}) + is None + ) + + def test_azure_coordinates_are_all_required(self): + from .stub_vault import jwt + + complete = { + "role": "warpgate", + "jwt": jwt({"aud": "https://management.azure.com/"}), + "subscription_id": "sub-1", + "resource_group_name": "rg-1", + "vm_name": "vm-1", + } + assert reject_login("azure", complete) is None + for field in ("subscription_id", "resource_group_name", "vm_name"): + assert reject_login("azure", {**complete, field: ""}), f"{field} not checked" + + +class TestControlsStillApply: + """A new authentication path is the classic place for an existing control to + go missing — the shape of GHSA-qmr2-wp96-h9ff.""" + + def test_an_unauthorized_user_never_reaches_the_issuer( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port, assign=False) + code, _ = connect(processes, cert_wg, user, target, timeout) + + assert code != 0 + assert stub_vault.signs == [], "a certificate was issued before authorization" + + def test_a_role_that_climbs_out_of_the_mount_never_leaves_the_process( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The role is put into the request path, and a URL is normalised before + it is sent: `../../auth/token/create` would arrive at a different Vault + API altogether, with the gateway's own token attached to it.""" + user, target = make_user_and_target( + api, cert_ssh_port, role="../../auth/token/create" + ) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert stub_vault.requests == [], "a request left the process" + + # Which only means something if a target that names a real role does + # reach Vault from this same gateway. + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + assert stub_vault.requests + + def test_no_certificate_is_issued_for_targets_using_other_auth( + self, processes, cert_wg, stub_vault, api, timeout, wg_c_ed25519_pubkey + ): + port = processes.start_ssh_server( + trusted_keys=[wg_c_ed25519_pubkey.read_text()] + ) + wait_port(port) + + wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user(sdk.CreateUserRequest(username=f"user-{uuid4()}")) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, wg_role.id) + target = api.create_target( + sdk.TargetDataRequest( + name=f"pubkey-{uuid4()}", + options=sdk.TargetOptions( + sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host="localhost", + port=port, + username="root", + auth=sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetPublicKeyAuth(kind="PublicKey") + ), + ) + ), + ) + ) + api.add_target_role(target.id, wg_role.id) + + connect(processes, cert_wg, user, target, timeout) + assert stub_vault.signs == [] + + +class TestSecrets: + def test_the_vault_token_is_never_logged( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + assert stub_vault.valid_token + log = Path(cert_wg.log_path).read_text() + # Asserted so the check below cannot pass merely because nothing was logged. + assert "Issued an SSH certificate" in log + assert stub_vault.valid_token not in log + + def test_no_credential_reaches_the_log_even_at_trace_level( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """Trace is where an operator goes when sessions will not connect, so it + is the setting most likely to be on while something is going wrong — and + the output most likely to be pasted into an issue. Neither the + credential Warpgate authenticates with nor the token it gets back may be + in it, at any verbosity.""" + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + log_path = ctx.tmpdir / f"trace-wg-{uuid4()}.log" + with log_path.open("w") as log: + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + }, + env={"RUST_LOG": "trace"}, + stdout=log, + stderr=log, + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, wg, user, target, timeout)[0] == 0 + + text = Path(log_path).read_text(errors="replace") + # Both halves of the exchange happened, so the log had the chance. + assert "Issued an SSH certificate" in text + assert stub_vault.logins and stub_vault.valid_token + + assert stub_vault.valid_token not in text, "the Vault token is in the log" + assert SERVICE_ACCOUNT_JWT not in text, "the service account token is in the log" + assert SERVICE_ACCOUNT_JWT.split(".")[1] not in text, "its claims are in the log" + + def test_no_ephemeral_key_is_stored( + self, processes, cert_wg, cert_ssh_port, api, timeout + ): + """The whole point of the feature: nothing the target would trust may + outlive the connection.""" + before = len(api.get_ssh_own_keys()) + + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + assert len(api.get_ssh_own_keys()) == before diff --git a/warpgate-aws/src/error.rs b/warpgate-aws/src/error.rs index fdac26d2a..03e9e014a 100644 --- a/warpgate-aws/src/error.rs +++ b/warpgate-aws/src/error.rs @@ -19,8 +19,13 @@ pub enum AwsError { ResourceNotFound(AwsResourceType, String), #[error("no AWS credentials available")] NoCredentials, + #[error( + "static AWS credentials (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY without a session token) are disallowed; Vault authentication requires a temporary workload identity (instance profile, pod identity, or IRSA)" + )] + StaticCredentialsDisallowed, #[error("credentials: {0}")] Credentials(#[from] CredentialsError), + #[error("signing parameters: {0}")] SigningParams(#[from] BuildError), #[error("signing: {0}")] @@ -35,4 +40,17 @@ impl AwsError { pub fn sdk_error(err: E) -> Self { Self::Other(Box::new(err)) } + + pub fn client_message(&self) -> &'static str { + match self { + AwsError::NoCredentials + | AwsError::StaticCredentialsDisallowed + | AwsError::Credentials(_) => "AWS credential provider error", + AwsError::SigningParams(_) | AwsError::Signing(_) | AwsError::Http(_) => { + "AWS signing request failed" + } + AwsError::RegionUnknown(_) | AwsError::ResourceNotFound(_, _) => "AWS resource error", + AwsError::Other(_) => "AWS integration error", + } + } } diff --git a/warpgate-aws/src/lib.rs b/warpgate-aws/src/lib.rs index 96c4c8d37..2c6cac5d4 100644 --- a/warpgate-aws/src/lib.rs +++ b/warpgate-aws/src/lib.rs @@ -9,6 +9,7 @@ mod error; mod rds; mod region; mod s3; +mod sts_identity; pub use ec2::{Ec2InstanceInfo, find_instance_by_ip, is_running_on_ec2, send_ssh_public_key}; pub use eks::{EksClusterInfo, find_eks_cluster_by_url, generate_eks_token}; @@ -19,6 +20,7 @@ pub use s3::{ AutoCredentials, S3Credentials, S3MultipartUpload, S3Storage, S3StorageConfig, StaticCredentials, }; +pub use sts_identity::{StsIdentityRequest, sign_sts_identity_request}; /// Cached EC2 detection result static EC2_DETECTION: OnceCell = OnceCell::const_new(); diff --git a/warpgate-aws/src/sts_identity.rs b/warpgate-aws/src/sts_identity.rs new file mode 100644 index 000000000..ebcd76492 --- /dev/null +++ b/warpgate-aws/src/sts_identity.rs @@ -0,0 +1,119 @@ +use std::collections::HashMap; +use std::time::SystemTime; + +use aws_credential_types::provider::ProvideCredentials; +use aws_sigv4::http_request::{SignableBody, SignableRequest, SigningSettings, sign}; +use aws_sigv4::sign::v4; +use tracing::debug; + +use crate::error::AwsError; + +const STS_BODY: &str = "Action=GetCallerIdentity&Version=2011-06-15"; + +/// A signed `sts:GetCallerIdentity` request, in the four parts a verifier needs +/// to replay it against STS and learn who signed it. +/// +/// The signature is what proves identity — no credential is disclosed — which is +/// why this can be handed to a third party such as Vault's AWS auth method. +#[derive(Debug)] +pub struct StsIdentityRequest { + pub method: &'static str, + pub url: String, + pub body: String, + pub headers: HashMap, +} + +/// Signs a `GetCallerIdentity` call with whatever the default credential chain +/// provides — on EC2 that is the instance role, whose credentials are short-lived +/// and never touch disk. Static access keys work too, but reintroduce exactly the +/// long-lived secret this authentication method exists to avoid. +/// +/// `server_id` is bound into the signature as the `X-Vault-AWS-IAM-Server-ID` +/// header when set, so a signed request captured by one verifier cannot be +/// replayed against another. +/// +/// `region` selects the regional STS endpoint. Leave it unset unless the verifier +/// is configured for one specific region: a signature scoped to a region is +/// rejected when the verifier replays it against the global endpoint, which is +/// what Vault does by default. +pub async fn sign_sts_identity_request( + region: Option<&str>, + server_id: Option<&str>, +) -> Result { + // The global endpoint expects signatures scoped to us-east-1. + let (host, signing_region) = match region { + Some(region) => (format!("sts.{region}.amazonaws.com"), region), + None => ("sts.amazonaws.com".to_owned(), "us-east-1"), + }; + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(aws_sdk_sts::config::Region::new(signing_region.to_string())) + .load() + .await; + + let credentials = config + .credentials_provider() + .ok_or(AwsError::NoCredentials)? + .provide_credentials() + .await?; + + if credentials.session_token().is_none() { + return Err(AwsError::StaticCredentialsDisallowed); + } + + let identity = credentials.into(); + + let url = format!("https://{host}/"); + + let mut headers = vec![ + ( + "content-type", + "application/x-www-form-urlencoded".to_owned(), + ), + ("host", host.clone()), + ]; + if let Some(server_id) = server_id { + headers.push(("x-vault-aws-iam-server-id", server_id.to_owned())); + } + + let signing_params = v4::SigningParams::builder() + .identity(&identity) + .region(signing_region) + .name("sts") + .time(SystemTime::now()) + .settings(SigningSettings::default()) + .build()?; + + let signable_request = SignableRequest::new( + "POST", + &url, + headers.iter().map(|(name, value)| (*name, value.as_str())), + SignableBody::Bytes(STS_BODY.as_bytes()), + )?; + + let (signing_instructions, _signature) = + sign(signable_request, &signing_params.into())?.into_parts(); + + let mut request = http::Request::builder().method("POST").uri(&url); + for (name, value) in &headers { + request = request.header(*name, value); + } + let mut request = request.body(())?; + signing_instructions.apply_to_request_http1x(&mut request); + + let headers = request + .headers() + .iter() + .filter_map(|(name, value)| { + Some((name.as_str().to_owned(), value.to_str().ok()?.to_owned())) + }) + .collect(); + + debug!(signing_region, "Signed an STS GetCallerIdentity request"); + + Ok(StsIdentityRequest { + method: "POST", + url, + body: STS_BODY.to_owned(), + headers, + }) +} diff --git a/warpgate-common/src/config/defaults.rs b/warpgate-common/src/config/defaults.rs index 6f36cac76..723c1a0f2 100644 --- a/warpgate-common/src/config/defaults.rs +++ b/warpgate-common/src/config/defaults.rs @@ -1,8 +1,33 @@ use std::net::{Ipv6Addr, SocketAddr}; +use std::path::PathBuf; use std::time::Duration; use crate::{ListenEndpoint, Secret}; +pub fn _default_vault_ssh_mount() -> String { + "ssh-client-signer".to_owned() +} + +pub const fn _default_vault_timeout() -> Duration { + Duration::from_secs(10) +} + +pub fn _default_vault_kubernetes_token_path() -> PathBuf { + PathBuf::from("/var/run/secrets/kubernetes.io/serviceaccount/token") +} + +pub fn _default_azure_resource() -> String { + "https://management.azure.com/".to_owned() +} + +pub fn _default_azure_imds() -> String { + "http://169.254.169.254".to_owned() +} + +pub fn _default_gcp_metadata() -> String { + "http://metadata.google.internal".to_owned() +} + pub const fn _default_true() -> bool { true } diff --git a/warpgate-common/src/config/mod.rs b/warpgate-common/src/config/mod.rs index 4b0e33280..bdbeac481 100644 --- a/warpgate-common/src/config/mod.rs +++ b/warpgate-common/src/config/mod.rs @@ -7,11 +7,13 @@ use std::path::PathBuf; use std::time::Duration; use defaults::{ - _default_audit_retention, _default_cookie_max_age, _default_database_url, _default_false, + _default_audit_retention, _default_azure_imds, _default_azure_resource, + _default_cookie_max_age, _default_database_url, _default_false, _default_gcp_metadata, _default_http_listen, _default_kubernetes_listen, _default_mysql_advertised_version, _default_mysql_listen, _default_postgres_listen, _default_rdp_listen, _default_recordings_path, _default_retention, _default_session_max_age, _default_ssh_inactivity_timeout, - _default_ssh_keys_path, _default_ssh_listen, _default_vnc_listen, + _default_ssh_keys_path, _default_ssh_listen, _default_vault_kubernetes_token_path, + _default_vault_ssh_mount, _default_vault_timeout, _default_vnc_listen, }; use poem_openapi::{Object, Union}; use schemars::JsonSchema; @@ -410,6 +412,96 @@ pub enum LogFormat { Json, } +/// How Warpgate proves its own identity to Vault. +/// +/// Both methods read their credential from a file rather than from the config, +/// deliberately: the point of issuing target credentials on demand is that +/// nothing long-lived sits on the Warpgate host, and a value pasted into the +/// config would put it straight back. A Kubernetes service account token is +/// mounted and rotated by the kubelet; an AppRole secret ID is short-lived and +/// meant to be delivered response-wrapped by whatever provisions the host. +#[derive(Debug, Deserialize, Serialize, Clone, JsonSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum VaultAuth { + Kubernetes { + role: String, + #[serde(default = "_default_vault_kubernetes_token_path")] + token_path: PathBuf, + }, + AppRole { + role_id: String, + secret_id_path: PathBuf, + }, + /// Signs an `sts:GetCallerIdentity` call with the default credential chain — + /// on EC2 that is the instance role. Access keys in the environment work but + /// put back the long-lived secret this exists to avoid. + Aws { + #[serde(default)] + role: Option, + /// Bound into the signature so a captured request cannot be replayed + /// against a different Vault. Must match the server's `iam_server_id_header_value`. + #[serde(default)] + server_id: Option, + /// Signs against a regional STS endpoint instead of the global one. Set + /// this only when Vault is configured with a matching `sts_endpoint`: + /// Vault replays the request globally by default, and a region-scoped + /// signature is rejected there. + #[serde(default)] + region: Option, + }, + Azure { + role: String, + #[serde(default = "_default_azure_resource")] + resource: String, + #[serde(default = "_default_azure_imds")] + metadata_address: String, + }, + Gcp { + role: String, + #[serde(default = "_default_gcp_metadata")] + metadata_address: String, + }, +} + +impl VaultAuth { + pub const fn kind(&self) -> &'static str { + match self { + Self::Kubernetes { .. } => "kubernetes", + Self::AppRole { .. } => "approle", + Self::Aws { .. } => "aws", + Self::Azure { .. } => "azure", + Self::Gcp { .. } => "gcp", + } + } +} + +#[derive(Debug, Deserialize, Serialize, Clone, JsonSchema)] +pub struct VaultConfig { + /// Base URL of the Vault server, e.g. `https://vault.internal:8200`. + pub address: String, + + /// Mount point of the SSH secrets engine in signed-certificates mode. + #[serde(default = "_default_vault_ssh_mount")] + pub mount: String, + + /// Signing role used by targets that don't name one of their own. + pub default_role: String, + + pub auth: VaultAuth, + + /// Lifetime asked for when signing. Vault clamps this to the role's + /// `max_ttl`, so it can only shorten what the role already allows — set it + /// to hold the window down without editing Vault. Left unset, the role's own + /// TTL decides. + #[serde(default, with = "humantime_serde::option")] + #[schemars(with = "Option")] + pub certificate_ttl: Option, + + #[serde(default = "_default_vault_timeout", with = "humantime_serde")] + #[schemars(with = "String")] + pub timeout: Duration, +} + #[derive(Debug, Deserialize, Serialize, Clone, JsonSchema)] pub struct SshConfig { #[serde(default = "_default_false")] @@ -883,6 +975,10 @@ pub struct WarpgateConfigStore { #[schemars(with = "String")] pub database_url: Secret, + /// Absent unless the deployment issues target credentials from Vault. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vault: Option, + #[serde(default)] pub ssh: SshConfig, @@ -915,6 +1011,7 @@ impl Default for WarpgateConfigStore { recordings: <_>::default(), external_host: None, database_url: _default_database_url(), + vault: None, ssh: <_>::default(), http: <_>::default(), kubernetes: <_>::default(), diff --git a/warpgate-common/src/config/target.rs b/warpgate-common/src/config/target.rs index 374396d01..5986820f3 100644 --- a/warpgate-common/src/config/target.rs +++ b/warpgate-common/src/config/target.rs @@ -51,6 +51,8 @@ pub enum SSHTargetAuth { Password(SshTargetPasswordAuth), #[serde(rename = "publickey")] PublicKey(SshTargetPublicKeyAuth), + #[serde(rename = "certificate")] + Certificate(SshTargetCertificateAuth), #[serde(rename = "iam_role")] IamRole(SshTargetIamRoleAuth), } @@ -68,6 +70,15 @@ pub struct SshTargetPublicKeyAuth { pub key_id: Option, } +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Object, Default)] +pub struct SshTargetCertificateAuth { + /// Vault signing role for this target; `None` uses the configured default + /// role. The role is what constrains which principals may be requested, so + /// targets of differing privilege belong to differing roles. + #[serde(default)] + pub role: Option, +} + #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Object, Default)] pub struct SshTargetIamRoleAuth {} diff --git a/warpgate-common/src/encryption.rs b/warpgate-common/src/encryption.rs index a1510a488..1a0d2edd7 100644 --- a/warpgate-common/src/encryption.rs +++ b/warpgate-common/src/encryption.rs @@ -406,10 +406,13 @@ mod tests { let (head, payload) = encrypted.rsplit_once(':').unwrap(); let mut bytes = BASE64.decode(payload.as_bytes()).unwrap(); - let last = bytes.len() - 1; - bytes.swap(0, last); + // Bitwise flip instead of swapping 0 and last bytes: random 12-byte GCM nonces + // can occasionally yield bytes[0] == bytes[last], making swap a no-op and causing + // false test failure. + bytes[0] ^= 0xff; let tampered = format!("{head}:{}", BASE64.encode(&bytes)); + assert!(matches!( k.decrypt(&tampered), Err(EncryptionError::Corrupt) diff --git a/warpgate-core/Cargo.toml b/warpgate-core/Cargo.toml index 772260bf6..465a407fa 100644 --- a/warpgate-core/Cargo.toml +++ b/warpgate-core/Cargo.toml @@ -51,6 +51,7 @@ warpgate-db-migrations = { path = "../warpgate-db-migrations" } warpgate-ldap = { path = "../warpgate-ldap" } warpgate-sso = { path = "../warpgate-sso", default-features = false } warpgate-tls = { path = "../warpgate-tls" } +warpgate-vault = { path = "../warpgate-vault" } webpki = { version = "0.22", default-features = false } zune-jpeg.workspace = true diff --git a/warpgate-core/src/services.rs b/warpgate-core/src/services.rs index e61fa04c4..615e0cad8 100644 --- a/warpgate-core/src/services.rs +++ b/warpgate-core/src/services.rs @@ -10,6 +10,7 @@ use tracing::warn; use warpgate_common::auth::{AuthState, CredentialKind}; use warpgate_common::{GlobalParams, Protocol, Secret, SessionId, WarpgateConfig, WarpgateError}; use warpgate_db_entities::Parameters; +use warpgate_vault::VaultClient; use crate::cluster::Cluster; use crate::db::connect_to_db_and_migrate; @@ -28,6 +29,8 @@ pub struct Services { pub cluster: Arc, pub state: Arc>, pub config_provider: Arc, + /// Present only when the config declares a Vault server. + pub vault: Option>, pub auth_state_store: Arc>, pub admin_token: Arc>>, pub cluster_token: Arc>, @@ -74,6 +77,14 @@ impl Services { let cluster = Arc::new(Cluster::new(db.clone(), config.store.http.listen.port()).await?); + let vault = config + .store + .vault + .clone() + .map(VaultClient::new) + .transpose()? + .map(Arc::new); + let config = Arc::new(Mutex::new(config)); let config_provider = Arc::new(DatabaseConfigProvider::new(&db).into()); @@ -124,6 +135,7 @@ impl Services { cluster, rate_limiter_registry, config_provider, + vault, auth_state_store, admin_token: Arc::new(admin_token.map(Secret::new)), cluster_token: Arc::new(resolve_cluster_token(&db).await?), diff --git a/warpgate-protocol-http/src/api/info.rs b/warpgate-protocol-http/src/api/info.rs index cb7eb77ad..fdb2e3590 100644 --- a/warpgate-protocol-http/src/api/info.rs +++ b/warpgate-protocol-http/src/api/info.rs @@ -132,6 +132,9 @@ pub struct Info { setup_state: Option, admin_permissions: Option, running_on_ec2: Option, + /// Whether a Vault server is configured, so the admin UI can hide target + /// authentication options that would fail at connect time. + has_vault: Option, should_prompt_analytics: bool, /// Login banner, shown to unauthenticated visitors too. Empty when unset. banner: String, @@ -355,6 +358,7 @@ impl Api { } else { None }, + has_vault: auth_ctx.is_some().then(|| ctx.services().vault.is_some()), should_prompt_analytics, banner: parameters.banner.clone(), show_session_menu: parameters.show_session_menu, diff --git a/warpgate-protocol-ssh/Cargo.toml b/warpgate-protocol-ssh/Cargo.toml index 83bfaca9a..d42a749bc 100644 --- a/warpgate-protocol-ssh/Cargo.toml +++ b/warpgate-protocol-ssh/Cargo.toml @@ -38,4 +38,5 @@ warpgate-common-http = { path = "../warpgate-common-http", default-features = fa warpgate-core = { path = "../warpgate-core", default-features = false } warpgate-db-entities = { path = "../warpgate-db-entities", default-features = false } warpgate-tls = { path = "../warpgate-tls" } +warpgate-vault = { path = "../warpgate-vault" } zeroize = { version = "^1.5", default-features = false } diff --git a/warpgate-protocol-ssh/src/client/mod.rs b/warpgate-protocol-ssh/src/client/mod.rs index 7113ca8b2..236671f5c 100644 --- a/warpgate-protocol-ssh/src/client/mod.rs +++ b/warpgate-protocol-ssh/src/client/mod.rs @@ -17,7 +17,8 @@ pub use error::SshClientError; use futures::{FutureExt, pin_mut}; use handler::ClientHandler; use russh::client::{AuthResult, Handle, KeyboardInteractiveAuthResponse}; -use russh::keys::{PrivateKeyWithHashAlg, PublicKey}; +use russh::keys::ssh_key::certificate::CertType; +use russh::keys::{Algorithm, Certificate, PrivateKey, PrivateKeyWithHashAlg, PublicKey}; use russh::{MethodKind, Preferred, Sig, kex, mac}; use serde::Serialize; use tokio::sync::mpsc::{ @@ -28,6 +29,7 @@ use tokio::task::JoinHandle; use tracing::*; use uuid::Uuid; use warpgate_aws::AwsError; +use warpgate_common::helpers::rng::get_crypto_rng; use warpgate_common::{SSHTargetAuth, SessionId, TargetOptions, TargetSSHOptions, WarpgateError}; use warpgate_core::{ConfigProvider, Services}; @@ -58,6 +60,9 @@ pub enum ConnectionError { #[error("AWS: {0}")] Aws(#[from] AwsError), + #[error("Vault: {0}")] + Vault(#[from] warpgate_vault::VaultError), + #[error("Could not resolve address")] Resolve, @@ -77,6 +82,41 @@ pub enum ConnectionError { Warpgate(#[from] WarpgateError), } +impl ConnectionError { + pub fn client_message(&self) -> String { + match self { + ConnectionError::Vault(e) => e.client_message().to_string(), + ConnectionError::Aws(e) => e.client_message().to_string(), + ConnectionError::Authentication => { + "SSH target rejected Warpgate's authentication request".to_string() + } + ConnectionError::HostKeyMismatch { .. } => "Host key mismatch".to_string(), + ConnectionError::Resolve => "Could not resolve target address".to_string(), + ConnectionError::Aborted => "Connection aborted".to_string(), + ConnectionError::Internal => "Internal connection error".to_string(), + ConnectionError::JumpHostTargetNotFound => "Jump host target not found".to_string(), + ConnectionError::Io(_) | ConnectionError::Key(_) | ConnectionError::Ssh(_) => { + "SSH protocol error".to_string() + } + ConnectionError::Warpgate(e) => e.to_string(), + } + } +} + +/// Vault is a second gate, not a trusted party, so what comes back is checked +/// against what was asked for before it is offered to the target. Neither of +/// these could authenticate anything anyway — the point is to say so in the +/// session log rather than let the target's refusal stand in for the reason. +fn certificate_mismatch(certificate: &Certificate, key: &PublicKey) -> Option<&'static str> { + if certificate.cert_type() != CertType::User { + return Some("Vault returned a host certificate rather than a user certificate"); + } + if certificate.public_key() != key.key_data() { + return Some("Vault signed a key other than the one generated for this session"); + } + None +} + pub struct ResolvedSshChainHost { pub name: String, pub ssh_options: TargetSSHOptions, @@ -795,6 +835,35 @@ impl RemoteClient { } } + /// Identifies the Warpgate session in the certificate's key ID. The target's + /// own sshd logs it verbatim, which is what lets a proxied session be traced + /// to a person from the target side alone. + async fn certificate_key_id(&self) -> String { + let session = self + .services + .state + .lock() + .await + .sessions + .get(&self.id) + .cloned(); + + let username = match session { + Some(session) => session + .lock() + .await + .user_info + .as_ref() + .map(|user| user.username.clone()), + None => None, + }; + + username.map_or_else( + || format!("warpgate:{}", self.id), + |username| format!("warpgate:{username}:{}", self.id), + ) + } + async fn authenticate_session( &self, session: &mut Handle, @@ -875,6 +944,67 @@ impl RemoteClient { Some("Public key authentication was rejected by the SSH target".into()); } } + SSHTargetAuth::Certificate(auth) => { + if let Some(vault) = self.services.vault.clone() { + // The key exists only for this authentication attempt — nothing + // the target will ever trust again outlives this scope. + let key = Arc::new( + PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .map_err(russh::keys::Error::from)?, + ); + let public_key = key + .public_key() + .to_openssh() + .map_err(russh::keys::Error::from)?; + let key_id = self.certificate_key_id().await; + let role = auth.role.as_deref().unwrap_or_else(|| vault.default_role()); + + let signed_key = vault + .sign_ssh_key(role, &public_key, username, &key_id) + .await?; + let certificate = + Certificate::from_openssh(&signed_key).map_err(russh::keys::Error::from)?; + + // The target's sshd enforces critical options, so one of + // these decides what the session actually does — a forced + // command runs instead of whatever the user asked for. + // Warpgate never requests any, so an operator has to be able + // to see that one arrived. + if !certificate.critical_options().is_empty() { + warn!( + options = ?certificate.critical_options().keys().collect::>(), + key_id, + "Vault issued a certificate carrying critical options" + ); + } + + if let Some(reason) = certificate_mismatch(&certificate, key.public_key()) { + auth_error_msg = Some(reason.into()); + } else { + let response = session + .authenticate_openssh_cert(username.to_string(), key, certificate) + .await?; + + auth_result = self + ._handle_auth_result(session, username.to_string(), response) + .await + .unwrap_or(false); + + if auth_result { + debug!( + username = username, + key_id, "Authenticated with certificate" + ); + } else { + auth_error_msg = Some( + "Certificate authentication was rejected by the SSH target".into(), + ); + } + } + } else { + auth_error_msg = Some("No Vault server is configured".into()); + } + } SSHTargetAuth::IamRole(_) => { let instance_info = warpgate_aws::find_instance_by_ip(host).await?; diff --git a/warpgate-protocol-ssh/src/server/session.rs b/warpgate-protocol-ssh/src/server/session.rs index d2820ce15..b839425d7 100644 --- a/warpgate-protocol-ssh/src/server/session.rs +++ b/warpgate-protocol-ssh/src/server/session.rs @@ -1040,8 +1040,12 @@ impl ServerSession { .await; } error => { + tracing::error!(%error, "Target connection failed"); let _ = self - .emit_pty_error(&format!("Target connection failed: {error}")) + .emit_pty_error(&format!( + "Target connection failed: {}", + error.client_message() + )) .await; } } diff --git a/warpgate-vault/Cargo.toml b/warpgate-vault/Cargo.toml new file mode 100644 index 000000000..d4c3ba84c --- /dev/null +++ b/warpgate-vault/Cargo.toml @@ -0,0 +1,27 @@ +[package] +name = "warpgate-vault" +version = "0.27.4" +edition = "2024" +license = "Apache-2.0" +publish = false + +[dependencies] +data-encoding.workspace = true +reqwest = { workspace = true, features = ["json"] } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["fs", "sync"] } +tracing.workspace = true +url.workspace = true +warpgate-aws = { path = "../warpgate-aws" } +warpgate-common = { path = "../warpgate-common" } +zeroize = "1.8" + +[dev-dependencies] +rcgen.workspace = true +rustls = { workspace = true, features = ["aws_lc_rs"] } +rustls-pki-types.workspace = true +tokio = { workspace = true, features = ["net", "time"] } +tokio-rustls = { workspace = true, features = ["aws-lc-rs"] } + diff --git a/warpgate-vault/README.md b/warpgate-vault/README.md new file mode 100644 index 000000000..e51900c6a --- /dev/null +++ b/warpgate-vault/README.md @@ -0,0 +1,343 @@ +# SSH certificate authentication for targets + +Warpgate can authenticate to an SSH target with a short-lived OpenSSH certificate +issued on demand by HashiCorp Vault, instead of a private key it stores. + +## What changes + +| | Stored key (`publickey`) | Vault certificate (`certificate`) | +|---|---|---| +| On the Warpgate host | a private key every target trusts, indefinitely | nothing a target would accept | +| On each target | an `authorized_keys` entry per gateway key | one `TrustedUserCAKeys` line, set once | +| Revoking access | edit `authorized_keys` on every host | stop issuing at Vault | +| Target's own audit log | one shared gateway identity | the Warpgate username and session | +| Adding a target | provision the key first | nothing | + +Warpgate generates an ed25519 keypair per connection, has Vault sign the public +half for the target's username, authenticates, and drops both. The private half +never reaches disk, the database, or a log. + +``` + ssh user:target@warpgate + │ + ▼ + ┌──────────┐ 1. workload identity ┌───────┐ + │ Warpgate │ ──────────────────────► │ Vault │ + │ │ ◄────────────────────── │ CA │ + └──────────┘ 2. certificate (2 min) └───────┘ + │ ▲ + │ 3. ephemeral key + certificate │ trusts + ▼ │ + ┌──────────┐ TrustedUserCAKeys ──────────┘ + │ target │ (no authorized_keys) + └──────────┘ +``` + +Vault is on the critical path of every session — see *Operational notes* below. + +## 1. Vault: the SSH CA + +```bash +vault secrets enable -path=ssh-client-signer ssh +vault write ssh-client-signer/config/ca generate_signing_key=true key_type=ed25519 + +vault write ssh-client-signer/roles/warpgate - <<'EOF' +{ + "key_type": "ca", + "allow_user_certificates": true, + "allow_user_key_ids": true, + "allowed_users": "deploy,app", + "default_user": "deploy", + "ttl": "2m", + "max_ttl": "5m", + "default_extensions": { "permit-pty": "" } +} +EOF + +vault policy write warpgate - <<'EOF' +path "ssh-client-signer/sign/warpgate" { + capabilities = ["update"] +} +EOF +``` + +Two settings deserve attention: + +- **`allowed_users`** is the real access boundary. Never `*`. Targets of differing + privilege belong to differing roles, selected per target in Warpgate. +- **`allow_user_key_ids`** must be `true`. Without it Vault rejects the request + outright, and every session would be attributed to the Warpgate service + identity instead of to a person. + +`default_extensions` grants the minimum. Add `permit-port-forwarding` or +`permit-agent-forwarding` only if your users need them. + +## 2. Warpgate's identity to Vault + +This decides whether the feature delivers what it promises. Every method reads +its credential from a file or a metadata service, never from `warpgate.yaml`. + +| Deployment | `kind` | What is on disk | +|---|---|---| +| Kubernetes | `kubernetes` | nothing durable — the kubelet mounts and rotates the token | +| EC2 | `aws` | nothing — the instance role's credentials are short-lived | +| GCE | `gcp` | nothing — the token comes from the metadata server | +| Azure VM | `azure` | nothing — the token comes from IMDS | +| Anything else | `app_role` | a short-lived secret ID, ideally response-wrapped | + +Common part of the config: + +```yaml +vault: + address: https://vault.internal:8200 + mount: ssh-client-signer # default + default_role: warpgate + timeout: 10s # default + certificate_ttl: 3m # optional; the role's TTL decides when unset + auth: + ... +``` + +`certificate_ttl` is a request, not a grant: Vault clamps it to the role's +`max_ttl`, so it can shorten the window but never widen it. It is there so the +lifetime can be tightened from Warpgate's side without an edit to Vault. + +Warpgate refuses a plain-HTTP `address` unless it is loopback, and verifies +Vault's certificate against the **host's trust store** — there is no CA-bundle +setting. A Vault behind a private CA therefore needs that CA installed on the +Warpgate host or in its container image (`/usr/local/share/ca-certificates` + +`update-ca-certificates` on Debian-based images), which is the same thing every +other outbound connection in Warpgate expects. + +### Kubernetes + +```yaml + auth: + kind: kubernetes + role: warpgate + # token_path defaults to the projected service account token +``` +```bash +vault auth enable kubernetes +vault write auth/kubernetes/config kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" +vault write auth/kubernetes/role/warpgate \ + bound_service_account_names=warpgate \ + bound_service_account_namespaces=warpgate \ + token_policies=warpgate ttl=1h +``` + +### AWS + +```yaml + auth: + kind: aws + role: warpgate + server_id: vault.internal # must match iam_server_id_header_value +``` +```bash +vault auth enable aws +vault write auth/aws/config/client iam_server_id_header_value=vault.internal +vault write auth/aws/role/warpgate auth_type=iam \ + bound_iam_principal_arn="arn:aws:iam:::role/" \ + token_policies=warpgate ttl=1h +``` + +Warpgate signs an `sts:GetCallerIdentity` request; Vault replays it against STS +to learn who signed it. No credential is disclosed. `server_id` is bound into the +signature so a captured request cannot be replayed against a different Vault. + +Leave `region` unset. Vault replays against the global endpoint, which rejects a +signature scoped to any other region. Set it only if Vault has a matching +`sts_endpoint`. + +### GCE + +```yaml + auth: + kind: gcp + role: warpgate +``` +```bash +gcloud services enable iam.googleapis.com # required, see Troubleshooting +vault auth enable gcp +vault write auth/gcp/config +vault write auth/gcp/role/warpgate type=gce \ + bound_projects= bound_zones= \ + token_policies=warpgate ttl=1h +``` + +### Azure + +```yaml + auth: + kind: azure + role: warpgate +``` +```bash +vault auth enable azure +vault write auth/azure/config \ + tenant_id= resource=https://management.azure.com/ \ + client_id= client_secret= +vault write auth/azure/role/warpgate \ + bound_subscription_ids= \ + bound_resource_groups= \ + bound_service_principal_ids= \ + token_policies=warpgate ttl=1h +``` + +The VM needs a system-assigned managed identity. Vault needs its own service +principal with `Reader` on the resource group to verify the VM through ARM. + +### AppRole + +```yaml + auth: + kind: app_role + role_id: + secret_id_path: /run/secrets/vault-secret-id +``` + +The secret ID is read fresh on every login, so it can be rotated underneath a +running Warpgate. + +## 3. Targets + +Add the CA public key and remove nothing else yet: + +```bash +vault read -field=public_key ssh-client-signer/config/ca | sudo tee /etc/ssh/trusted-ca.pub +sudo chmod 600 /etc/ssh/trusted-ca.pub +echo 'TrustedUserCAKeys /etc/ssh/trusted-ca.pub' | sudo tee -a /etc/ssh/sshd_config +sudo sshd -t && sudo systemctl reload sshd +``` + +The target now accepts certificates from this CA *in addition* to whatever it +accepted before. Once certificate auth is confirmed working, remove the gateway's +entry from `authorized_keys`. + +For a second boundary on the target side, `AuthorizedPrincipalsFile` restricts +which principals may log in as which local user. + +## 4. Warpgate target configuration + +In the admin UI, an SSH target's **Authenticate using** now offers *Certificate +issued by Vault*, with an optional signing role. The option appears only when +`vault:` is configured. Leave the role empty to use `default_role`. + +Through the API: + +```json +{ + "name": "prod-db", + "options": { + "kind": "Ssh", "host": "db.internal", "port": 22, "username": "deploy", + "auth": { "kind": "Certificate", "role": "warpgate-prod" } + } +} +``` + +## Verification + +```bash +# Warpgate: one login, one certificate per session +grep -E 'Authenticated to Vault|Issued an SSH certificate' + +# The target names the person, not the gateway +grep 'Accepted certificate ID' /var/log/auth.log + +# Nothing was persisted: only the bootstrap keys, whatever you do +# (admin UI → Config → SSH, or GET /@warpgate/admin/api/ssh/own-keys) +``` + +Expected in the target's log: + +``` +Accepted certificate ID "warpgate:alice:8fae5e3d-..." (serial ...) signed by ED25519 CA ... +``` + +## Troubleshooting + +**`SignatureDoesNotMatch: Credential should be scoped to a valid region`** (AWS) — +`region` is set in the config but Vault replays against the global STS endpoint. +Remove it, or set Vault's `sts_endpoint` to match. + +**`SERVICE_DISABLED ... Identity and Access Management (IAM) API`** (GCP) — Vault +resolves the service account named in the identity token through the IAM API. +`gcloud services enable iam.googleapis.com`, then wait a minute. + +**`expected specific bound_group_ids or bound_service_principal_ids`** (Azure) — +the Vault role binds only a subscription and resource group. Add +`bound_service_principal_ids`, the object ID of the VM's managed identity (the +`oid` claim of the IMDS token). + +**A policy change appears to have no effect** — Warpgate caches the Vault token +until shortly before its lease expires. A role or policy change does not +invalidate a token already issued. Restart Warpgate, or wait out `token_ttl`. + +**`Certificate authentication was rejected by the SSH target`** — the certificate +was issued but the target refused it. Check `sshd -T | grep trustedusercakeys`, +that the principal matches the target username, and the clock skew between +Warpgate and the target. + +**`No Vault server is configured`** — the target uses certificate auth but +`vault:` is absent from the config. + +## OpenBao Compatibility + +Warpgate's Vault integration is fully compatible with **OpenBao**. The SSH secrets engine API and `/sign/` endpoints are identical. + +Note for OpenBao deployments: +- AWS, Azure, and GCP auth engines are not bundled in OpenBao's core binary; they are separate plugins (`openbao/openbao-plugins`) that must be registered and mounted before those cloud login paths can be used. +- Kubernetes and AppRole auth engines are built-in to OpenBao out of the box. +- For AppRole authentication, Warpgate supports both static Secret IDs and Vault Response Wrapping tokens (specify `unwrap:` in the credential file). + +## Operational notes + +**Vault is on the critical path of every session.** An outage locks every target +using certificate auth — including the one you would use to reach Vault. Keep a +break-glass path: one target on stored-key auth, or console access. + +**Clock skew.** Certificates live minutes. A target whose clock lags rejects them +as not yet valid, with an error that does not say so. Require NTP on targets. + +**Certificate TTL vs. session length.** The TTL bounds how long the certificate +may be *presented*, not how long the session lasts. An established session +survives expiry. + +**Revocation.** Stopping issuance at Vault takes effect for new sessions only. An +already-issued certificate stays valid for its TTL — which is why the TTL is +minutes. + +**Warpgate does not rate-limit issuance.** One session is one signing call, and +nothing on the Warpgate side caps how many a user can start. Put the ceiling in +Vault — a rate limit quota on the mount (`vault write sys/quotas/rate-limit/ssh +path=ssh-client-signer rate=...`) bounds it without depending on Warpgate +behaving. + +**A slow Vault delays every session, not just one.** Warpgate holds one client +token and logs in once for everybody; the login is serialised so an expiring +token cannot send every session in flight to Vault at once. The cost is that a +login which hangs blocks the others for up to `timeout`. Keep `timeout` at a +value you are willing to make every concurrent session wait. + +**A certificate can come back carrying critical options.** If a Vault role sets +`default_critical_options`, a `force-command` there replaces whatever the user +typed — the target runs Vault's command and the session recording shows its +output. Warpgate does not refuse such a certificate, because a restricted role +may set one deliberately, but it logs every arrival: + +``` +WARN Vault issued a certificate carrying critical options options=["force-command"] +``` + +Alert on that line if your roles are not supposed to set any. + +**`address` and `metadata_address` are fetched as given.** Both come from +`warpgate.yaml`, so anyone who can edit that file can point Warpgate's outbound +requests — including the ones carrying its Vault token and its cloud identity +token — at a host of their choosing. That is no more access than editing the +config already grants, but it means the config file deserves the same protection +as a credential: root-owned, not world-readable, and under change review. +Warpgate refuses plain HTTP for `address` outside localhost and never follows a +redirect, so neither address can be downgraded or re-pointed after the fact by +whatever answers it. diff --git a/warpgate-vault/src/client.rs b/warpgate-vault/src/client.rs new file mode 100644 index 000000000..75678447f --- /dev/null +++ b/warpgate-vault/src/client.rs @@ -0,0 +1,1003 @@ +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use data_encoding::BASE64; +use reqwest::StatusCode; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; +use tracing::{debug, warn}; +use warpgate_common::{VaultAuth, VaultConfig}; +use zeroize::{Zeroize, Zeroizing}; + +use crate::error::{Result, VaultError}; +use crate::metadata; + +/// Re-login this long before the lease actually runs out, so a certificate +/// request issued just after the check cannot race the expiry. +const TOKEN_EXPIRY_MARGIN: Duration = Duration::from_secs(30); + +/// How much of a failed response is kept for the error message. Anything +/// answering on the configured address can send an arbitrarily large body, so +/// this bounds what a bad endpoint can make Warpgate allocate. +const MAX_ERROR_BODY: usize = 256; + +/// The largest response body accepted from a successful call. A signed +/// certificate is a couple of kilobytes and an auth response smaller still, so +/// the bound is far above anything Vault sends — but without one, an endpoint +/// answering on the configured address can make every session in flight buffer +/// a body of its choosing. +const MAX_RESPONSE_BODY: usize = 256 * 1024; + +static NEXT_TOKEN_ID: AtomicU64 = AtomicU64::new(1); + +fn validate_address(address: &str) -> Result<()> { + let parsed = url::Url::parse(address).map_err(|e| VaultError::InvalidAddress(e.to_string()))?; + if parsed.scheme() != "https" { + let host = parsed.host_str().unwrap_or_default(); + let is_localhost = host == "localhost" || host == "127.0.0.1" || host == "::1"; + if !is_localhost { + return Err(VaultError::InsecureAddress); + } + } + Ok(()) +} + +fn validate_segment(name: &str) -> Result<()> { + if name.is_empty() + || !name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') + { + return Err(VaultError::InvalidRole(name.to_string())); + } + Ok(()) +} + +/// Vault reads `valid_principals` as a comma-separated list, so a comma in the +/// target's username would silently widen the certificate to accounts the +/// operator never named. +fn validate_principal(principal: &str) -> Result<()> { + if principal.is_empty() || principal.contains(',') || principal.chars().any(char::is_control) { + return Err(VaultError::InvalidPrincipal(principal.to_owned())); + } + Ok(()) +} + +/// The key ID is echoed verbatim into the target's own sshd log, so a control +/// character in it would let a Warpgate username forge log lines there. +fn validate_key_id(key_id: &str) -> Result<()> { + if key_id.chars().any(char::is_control) { + return Err(VaultError::InvalidKeyId); + } + Ok(()) +} + +/// Keeps the error message bounded and always a valid string: `from_utf8_lossy` +/// replaces a multi-byte character cut in half by the byte limit instead of +/// panicking the way slicing a `String` at a non-boundary would. +fn render_error_body(bytes: &[u8], truncated: bool) -> String { + let mut text = String::from_utf8_lossy(bytes).into_owned(); + if truncated { + text.push_str("... (truncated)"); + } + text +} + +struct CachedToken { + id: u64, + value: Zeroizing, + /// `None` for a token Vault says has no lease. Revocation is still handled: + /// a rejected token is dropped when signing comes back `403`. + expires_at: Option, +} + +fn login_payload(value: &T) -> Result> { + Ok(Zeroizing::new(serde_json::to_string(value)?)) +} + +#[derive(Serialize)] +struct JwtLogin<'a> { + role: &'a str, + jwt: &'a str, +} + +#[derive(Serialize)] +struct AppRoleLogin<'a> { + role_id: &'a str, + secret_id: &'a str, +} + +#[derive(Serialize)] +struct AzureLogin<'a> { + role: &'a str, + jwt: &'a str, + subscription_id: &'a str, + resource_group_name: &'a str, + vm_name: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + vmss_name: Option<&'a str>, +} + +#[derive(Serialize)] +struct AwsLogin<'a> { + role: Option<&'a str>, + iam_http_request_method: &'a str, + iam_request_url: String, + iam_request_body: String, + iam_request_headers: String, +} + +#[derive(Serialize)] +struct SignRequest<'a> { + public_key: &'a str, + valid_principals: &'a str, + cert_type: &'a str, + key_id: &'a str, + /// Omitted rather than sent as zero when unset: Vault reads a zero TTL as + /// "use the role's default", but only if the field is absent. + #[serde(skip_serializing_if = "Option::is_none")] + ttl: Option, +} + +#[derive(Deserialize)] +struct AuthResponse { + auth: AuthData, +} + +#[derive(Deserialize)] +struct AuthData { + client_token: String, + lease_duration: u64, +} + +#[derive(Deserialize)] +struct SignResponse { + data: SignData, +} + +#[derive(Deserialize)] +struct SignData { + signed_key: String, +} + +#[derive(Deserialize)] +struct UnwrapResponse { + data: Option, +} + +#[derive(Deserialize)] +struct UnwrapData { + secret_id: Option, +} + +pub struct VaultClient { + config: VaultConfig, + http: reqwest::Client, + token: Mutex>, +} + +impl VaultClient { + pub fn new(config: VaultConfig) -> Result { + validate_address(&config.address)?; + validate_segment(&config.mount)?; + validate_segment(&config.default_role)?; + + if !config.address.starts_with("https://") { + // Only reachable for a loopback address, which validation allows so + // a development Vault does not need a certificate. Said out loud + // because the token crosses this connection in a header. + warn!( + address = config.address, + "The Vault address is not HTTPS; the client token is sent in clear text" + ); + } + + // Redirects are refused rather than followed: reqwest strips + // `Authorization` on a cross-origin hop but knows nothing about + // `X-Vault-Token`, so a 307 from a hostile or misconfigured endpoint + // would replay the token to another host, or downgrade the request to + // plain HTTP. The same client fetches cloud metadata, where following a + // redirect would be just as wrong. + let http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(config.timeout) + .build()?; + Ok(Self { + config, + http, + token: Mutex::new(None), + }) + } + + pub fn default_role(&self) -> &str { + &self.config.default_role + } + + /// Signs `public_key` into a short-lived OpenSSH user certificate, returned + /// in OpenSSH wire format. + /// + /// `principals` are usernames on the target, not Warpgate usernames. `key_id` + /// is echoed into the target's own sshd log, which is what makes a session + /// attributable to a person rather than to the gateway. + pub async fn sign_ssh_key( + &self, + role: &str, + public_key: &str, + principals: &str, + key_id: &str, + ) -> Result { + validate_segment(role)?; + validate_principal(principals)?; + validate_key_id(key_id)?; + + let (token_id, result) = self.sign_once(role, public_key, principals, key_id).await; + + match result { + // A cached token can stop being accepted long before its lease runs + // out — it may have been revoked, or Vault resealed or restarted. + // Forcing one re-login tells that apart from a real policy denial. + // Only invalidate if the token in cache is still the token that failed. + Err(VaultError::Api { status, .. }) if status == StatusCode::FORBIDDEN => { + let mut guard = self.token.lock().await; + if guard.as_ref().map(|t| t.id) == Some(token_id) { + *guard = None; + } + drop(guard); + let (_, second_try) = self.sign_once(role, public_key, principals, key_id).await; + second_try + } + res => res, + } + } + + async fn sign_once( + &self, + role: &str, + public_key: &str, + principals: &str, + key_id: &str, + ) -> (u64, Result) { + let (token_id, token) = match self.token().await { + Ok(t) => t, + Err(err) => return (0, Err(err)), + }; + + let response = self + .http + .post(self.url(&format!("{}/sign/{role}", self.config.mount))) + .header("X-Vault-Token", token.as_str()) + .json(&SignRequest { + public_key, + valid_principals: principals, + cert_type: "user", + key_id, + ttl: self + .config + .certificate_ttl + .map(|ttl| format!("{}s", ttl.as_secs())), + }) + .send() + .await; + + let res = match response { + Ok(resp) => match Self::check(resp).await { + Ok(resp) => match Self::read_json::(resp).await { + Ok(parsed) => { + debug!(role, principals, key_id, "Issued an SSH certificate"); + Ok(parsed.data.signed_key) + } + Err(e) => Err(e), + }, + Err(e) => Err(e), + }, + Err(e) => Err(VaultError::Request(e)), + }; + + (token_id, res) + } + + /// The cached client token, logging in again once it nears expiry. Plain + /// re-login is used instead of Vault's renew API because it behaves the same + /// whether or not the token has hit its `max_ttl`. + /// + /// The lock is deliberately held across the login: one expiring token would + /// otherwise send every session in flight to Vault at once, and for the + /// cloud methods each of those first fetches an identity token of its own. + /// The cost of that choice is that a slow login delays every other session + /// by up to `timeout`. + async fn token(&self) -> Result<(u64, Zeroizing)> { + let mut cached = self.token.lock().await; + + if let Some(token) = cached.as_ref() + && token.expires_at.is_none_or(|at| Instant::now() < at) + { + return Ok((token.id, token.value.clone())); + } + + let token = self.login().await?; + let id = token.id; + let value = token.value.clone(); + *cached = Some(token); + Ok((id, value)) + } + + async fn login(&self) -> Result { + let (path, body) = self.login_body().await?; + + let response = self + .http + .post(self.url(path)) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(body.as_bytes().to_vec()) + .send() + .await?; + let auth = Self::read_json::(Self::check(response).await?) + .await? + .auth; + + debug!(method = self.config.auth.kind(), "Authenticated to Vault"); + + let token_id = NEXT_TOKEN_ID.fetch_add(1, Ordering::Relaxed); + Ok(CachedToken { + id: token_id, + value: Zeroizing::new(auth.client_token), + // Vault reports a lease of zero for a token that does not expire. + // Reading that as "expired half a minute ago" would turn every + // certificate request into a fresh login. + expires_at: (auth.lease_duration > 0).then(|| { + Instant::now() + + Duration::from_secs(auth.lease_duration).saturating_sub(TOKEN_EXPIRY_MARGIN) + }), + }) + } + + /// The endpoint and JSON payload for the configured authentication method. + /// + /// The payload is built through `serde` straight into a `Zeroizing` buffer + /// rather than through a `serde_json::Value`: every intermediate copy of a + /// service account token, secret ID or identity token is one more place the + /// credential outlives the request in freed memory. The buffer reqwest takes + /// to put the body on the wire is the one copy Warpgate cannot reach. + async fn login_body(&self) -> Result<(&'static str, Zeroizing)> { + Ok(match &self.config.auth { + VaultAuth::Kubernetes { role, token_path } => { + let jwt = Self::read_credential(token_path).await?; + ( + "auth/kubernetes/login", + login_payload(&JwtLogin { role, jwt: &jwt })?, + ) + } + VaultAuth::AppRole { + role_id, + secret_id_path, + } => { + let cred = Self::read_credential(secret_id_path).await?; + let secret_id = if cred.starts_with("unwrap:") { + let wrapping_token = + Zeroizing::new(cred.trim_start_matches("unwrap:").trim().to_owned()); + self.unwrap_secret_id(&wrapping_token).await? + } else { + cred + }; + ( + "auth/approle/login", + login_payload(&AppRoleLogin { + role_id, + secret_id: &secret_id, + })?, + ) + } + VaultAuth::Aws { + role, + server_id, + region, + } => ( + "auth/aws/login", + self.aws_login_body(role.as_deref(), server_id.as_deref(), region.as_deref()) + .await?, + ), + VaultAuth::Azure { + role, + resource, + metadata_address, + } => { + let (jwt, instance) = + metadata::azure_login_material(&self.http, metadata_address, resource).await?; + ( + "auth/azure/login", + login_payload(&AzureLogin { + role, + jwt: &jwt, + subscription_id: &instance.subscription_id, + resource_group_name: &instance.resource_group_name, + vm_name: &instance.name, + vmss_name: (!instance.vm_scale_set_name.is_empty()) + .then_some(instance.vm_scale_set_name.as_str()), + })?, + ) + } + VaultAuth::Gcp { + role, + metadata_address, + } => { + let audience = format!("vault/{role}"); + let jwt = + metadata::gcp_identity_token(&self.http, metadata_address, &audience).await?; + ( + "auth/gcp/login", + login_payload(&JwtLogin { role, jwt: &jwt })?, + ) + } + }) + } + + async fn unwrap_secret_id(&self, wrapping_token: &str) -> Result> { + let response = self + .http + .post(self.url("sys/wrapping/unwrap")) + .header("X-Vault-Token", wrapping_token) + .send() + .await?; + + let unwrap_resp = Self::read_json::(Self::check(response).await?).await?; + + // Moved rather than copied out of the response, so the only allocation + // holding the secret ID is the one that gets zeroized on drop. + unwrap_resp + .data + .and_then(|data| data.secret_id) + .map_or_else( + || { + Err(VaultError::Api { + status: StatusCode::BAD_REQUEST, + body: "failed to unwrap secret_id from Vault response".to_owned(), + }) + }, + |secret_id| Ok(Zeroizing::new(secret_id)), + ) + } + + async fn aws_login_body( + &self, + role: Option<&str>, + server_id: Option<&str>, + region: Option<&str>, + ) -> Result> { + let request = warpgate_aws::sign_sts_identity_request(region, server_id).await?; + // Carries the signature and, on an instance role, the session token. + let mut headers = serde_json::to_string(&request.headers)?; + + let payload = login_payload(&AwsLogin { + role, + iam_http_request_method: request.method, + iam_request_url: BASE64.encode(request.url.as_bytes()), + iam_request_body: BASE64.encode(request.body.as_bytes()), + iam_request_headers: BASE64.encode(headers.as_bytes()), + }); + headers.zeroize(); + payload + } + + async fn read_credential(path: &Path) -> Result> { + let raw = Zeroizing::new(tokio::fs::read_to_string(path).await.map_err(|source| { + VaultError::CredentialFile { + path: path.to_owned(), + source, + } + })?); + Ok(Zeroizing::new(raw.trim().to_owned())) + } + + fn url(&self, path: &str) -> String { + format!("{}/v1/{path}", self.config.address.trim_end_matches('/')) + } + + /// Parses a successful response, refusing one larger than any Vault answer + /// could be. `json()` would buffer whatever arrives before anything got to + /// look at its size, so a single endpoint could hold as much of Warpgate's + /// memory as it cared to send, once per session in flight. + async fn read_json( + mut response: reqwest::Response, + ) -> Result { + // A login response carries the client token and an unwrap response the + // secret ID, so the buffer they land in is zeroized rather than merely + // dropped. + let mut buf: Zeroizing> = Zeroizing::new(Vec::new()); + while let Some(chunk) = response.chunk().await? { + if buf.len() + chunk.len() > MAX_RESPONSE_BODY { + return Err(VaultError::OversizedResponse); + } + buf.extend_from_slice(&chunk); + } + Ok(serde_json::from_slice(&buf)?) + } + + async fn check(response: reqwest::Response) -> Result { + let status = response.status(); + if status.is_success() { + return Ok(response); + } + Err(VaultError::Api { + status, + body: Self::error_body(response).await, + }) + } + + /// The first `MAX_ERROR_BODY` bytes of a failed response. Read chunk by + /// chunk rather than with `text()`, which would buffer the whole body before + /// there was anything to truncate — an endpoint answering on the configured + /// address could then make an error message cost arbitrary memory. + async fn error_body(mut response: reqwest::Response) -> String { + let mut buf: Vec = Vec::new(); + let mut truncated = false; + + while buf.len() < MAX_ERROR_BODY { + match response.chunk().await { + Ok(Some(chunk)) => { + let room = MAX_ERROR_BODY.saturating_sub(buf.len()); + let taken = chunk.get(..room).unwrap_or(&chunk); + truncated |= taken.len() < chunk.len(); + buf.extend_from_slice(taken); + } + Ok(None) | Err(_) => break, + } + } + + // The loop can fill the buffer exactly while the body continues; one + // more poll is what tells a 256-byte body from a truncated one. + if !truncated && buf.len() == MAX_ERROR_BODY { + truncated = matches!(response.chunk().await, Ok(Some(chunk)) if !chunk.is_empty()); + } + + render_error_body(&buf, truncated) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::sync::{Arc, Mutex as StdMutex}; + + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + use super::*; + + /// A one-shot HTTP server that records the raw request it was sent. + /// + /// `login` answers the authentication endpoint, `other` everything else, so + /// a test can let the client reach the point where it holds a token. + async fn spawn_server( + login: String, + other: String, + log: Arc>>, + ) -> Result { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = format!("http://{}", listener.local_addr().unwrap()); + + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let mut buf = vec![0u8; 8192]; + let read = socket.read(&mut buf).await.unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..read]).into_owned(); + let response = if request.contains("/login") { + login.clone() + } else { + other.clone() + }; + log.lock().unwrap().push(request); + let _ = socket.write_all(response.as_bytes()).await; + let _ = socket.shutdown().await; + } + }); + + Ok(address) + } + + /// Answers the login endpoint normally and then streams an error body that + /// never ends — the shape of a hostile or broken endpoint. + async fn spawn_endless_error_server(login: String) -> String { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = format!("http://{}", listener.local_addr().unwrap()); + + tokio::spawn(async move { + while let Ok((mut socket, _)) = listener.accept().await { + let mut buf = vec![0u8; 8192]; + let read = socket.read(&mut buf).await.unwrap_or(0); + let request = String::from_utf8_lossy(&buf[..read]).into_owned(); + + if request.contains("/login") { + let _ = socket.write_all(login.as_bytes()).await; + let _ = socket.shutdown().await; + continue; + } + + let _ = socket + .write_all( + b"HTTP/1.1 500 Internal Server Error\r\nTransfer-Encoding: chunked\r\n\r\n", + ) + .await; + // Paced, so a reader that waits for the end cannot finish + // before the request timeout no matter how fast the loopback is. + let chunk = format!("1000\r\n{}\r\n", "a".repeat(4096)); + for _ in 0..1000 { + if socket.write_all(chunk.as_bytes()).await.is_err() { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + } + }); + + address + } + + fn json_response(body: &str) -> String { + format!( + "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}", + body.len() + ) + } + + fn approle_config(address: String, secret_id_path: PathBuf) -> VaultConfig { + VaultConfig { + address, + mount: "ssh-client-signer".to_owned(), + default_role: "warpgate".to_owned(), + auth: VaultAuth::AppRole { + role_id: "role-1".to_owned(), + secret_id_path, + }, + certificate_ttl: None, + timeout: Duration::from_secs(5), + } + } + + /// reqwest strips `Authorization` on a cross-origin redirect but has no idea + /// `X-Vault-Token` is a credential, so following one would hand the token to + /// whichever host the redirect names. + #[tokio::test] + async fn test_a_redirect_never_carries_the_token_to_another_host() { + // The binary does this in `main`; a unit test has to do it itself before + // reqwest will build a client at all. + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let attacker_log = Arc::new(StdMutex::new(vec![])); + let attacker = spawn_server( + json_response("{}"), + json_response("{}"), + attacker_log.clone(), + ) + .await + .unwrap(); + + let vault_log = Arc::new(StdMutex::new(vec![])); + let vault = spawn_server( + json_response(r#"{"auth":{"client_token":"s.stub-token","lease_duration":3600}}"#), + format!("HTTP/1.1 307 Temporary Redirect\r\nLocation: {attacker}/v1/steal\r\nContent-Length: 0\r\n\r\n"), + vault_log.clone(), + ) + .await + .unwrap(); + + let secret_id_path = std::env::temp_dir().join("warpgate-vault-redirect-test-secret"); + std::fs::write(&secret_id_path, "secret-id").unwrap(); + + let client = VaultClient::new(approle_config(vault, secret_id_path)).unwrap(); + let error = client + .sign_ssh_key("warpgate", "ssh-ed25519 AAAA", "root", "warpgate:alice") + .await + .unwrap_err(); + + assert!( + matches!(error, VaultError::Api { status, .. } if status == StatusCode::TEMPORARY_REDIRECT), + "the redirect should surface as an error, got {error:?}" + ); + assert!( + attacker_log.lock().unwrap().is_empty(), + "the redirect target was contacted at all" + ); + // Without this the test would also pass if the signing request had never + // been made, which is the failure mode it exists to rule out. + assert!( + vault_log + .lock() + .unwrap() + .iter() + .any(|request| request.to_lowercase().contains("x-vault-token")), + "no token-bearing request was made, so nothing was under test" + ); + } + + /// `text()` would buffer the whole body before there was anything to + /// truncate, so an endpoint that never stops sending holds the session open + /// until the request timeout and allocates everything it sent meanwhile. + #[tokio::test] + async fn test_an_endless_error_body_is_not_buffered() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let vault = spawn_endless_error_server(json_response( + r#"{"auth":{"client_token":"s.stub-token","lease_duration":3600}}"#, + )) + .await; + + let secret_id_path = std::env::temp_dir().join("warpgate-vault-endless-test-secret"); + std::fs::write(&secret_id_path, "secret-id").unwrap(); + + let mut config = approle_config(vault, secret_id_path); + config.timeout = Duration::from_secs(10); + let client = VaultClient::new(config).unwrap(); + + let started = Instant::now(); + let error = client + .sign_ssh_key("warpgate", "ssh-ed25519 AAAA", "root", "warpgate:alice") + .await + .unwrap_err(); + let elapsed = started.elapsed(); + + match error { + VaultError::Api { status, body } => { + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert!(body.len() <= MAX_ERROR_BODY + "... (truncated)".len()); + } + other => panic!("expected a bounded API error, got {other:?}"), + } + assert!( + elapsed < Duration::from_secs(2), + "reading the error body waited on the whole stream ({elapsed:?})" + ); + } + + /// The error path was bounded in an earlier round; the success path was not. + /// A body is a body whatever the status code on it says, and this one is + /// parsed for every session that asks for a certificate. + #[tokio::test] + async fn test_an_oversized_success_body_is_refused_rather_than_buffered() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let oversized = format!( + r#"{{"data":{{"signed_key":"{}"}}}}"#, + "A".repeat(MAX_RESPONSE_BODY + 1) + ); + let log = Arc::new(StdMutex::new(vec![])); + let vault = spawn_server( + json_response(r#"{"auth":{"client_token":"s.stub-token","lease_duration":3600}}"#), + json_response(&oversized), + log.clone(), + ) + .await + .unwrap(); + + let secret_id_path = std::env::temp_dir().join("warpgate-vault-oversized-test-secret"); + std::fs::write(&secret_id_path, "secret-id").unwrap(); + + let client = VaultClient::new(approle_config(vault, secret_id_path)).unwrap(); + let error = client + .sign_ssh_key("warpgate", "ssh-ed25519 AAAA", "root", "warpgate:alice") + .await + .unwrap_err(); + + assert!( + matches!(error, VaultError::OversizedResponse), + "expected the body to be refused on size, got {error:?}" + ); + // Otherwise this passes just as well when the request was never sent. + assert!( + log.lock().unwrap().iter().any(|r| r.contains("/sign/")), + "no signing request was made, so nothing was under test" + ); + } + + /// Vault reports `lease_duration: 0` for a token with no lease at all — a + /// root or otherwise non-expiring token. Treating that as an expiry in the + /// past makes every certificate request log in again, which is a login storm + /// against Vault sourced from ordinary traffic. + #[tokio::test] + async fn test_a_token_with_no_lease_is_not_re_fetched_for_every_request() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let log = Arc::new(StdMutex::new(vec![])); + let vault = spawn_server( + json_response(r#"{"auth":{"client_token":"s.stub-token","lease_duration":0}}"#), + json_response(r#"{"data":{"signed_key":"ssh-ed25519-cert-v01@openssh.com AAAA"}}"#), + log.clone(), + ) + .await + .unwrap(); + + let secret_id_path = std::env::temp_dir().join("warpgate-vault-no-lease-test-secret"); + std::fs::write(&secret_id_path, "secret-id").unwrap(); + + let client = VaultClient::new(approle_config(vault, secret_id_path)).unwrap(); + for _ in 0..3 { + client + .sign_ssh_key("warpgate", "ssh-ed25519 AAAA", "root", "warpgate:alice") + .await + .unwrap(); + } + + let logins = log + .lock() + .unwrap() + .iter() + .filter(|request| request.contains("/login")) + .count(); + assert_eq!( + logins, 1, + "the cached token was thrown away between requests" + ); + } + + /// A `200` is not a certificate. Vault answering with a body that parses but + /// carries no key must be an error, not an empty certificate handed to the + /// SSH layer to make sense of. + #[tokio::test] + async fn test_a_success_without_a_signed_key_is_an_error() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let log = Arc::new(StdMutex::new(vec![])); + let vault = spawn_server( + json_response(r#"{"auth":{"client_token":"s.stub-token","lease_duration":3600}}"#), + json_response(r#"{"data":{}}"#), + log.clone(), + ) + .await + .unwrap(); + + let secret_id_path = std::env::temp_dir().join("warpgate-vault-no-key-test-secret"); + std::fs::write(&secret_id_path, "secret-id").unwrap(); + + let client = VaultClient::new(approle_config(vault, secret_id_path)).unwrap(); + let error = client + .sign_ssh_key("warpgate", "ssh-ed25519 AAAA", "root", "warpgate:alice") + .await + .unwrap_err(); + + assert!( + matches!(error, VaultError::Json(_)), + "expected the response to be rejected, got {error:?}" + ); + assert!( + log.lock().unwrap().iter().any(|r| r.contains("/sign/")), + "no signing request was made, so nothing was under test" + ); + } + + /// Nothing else in the crate would notice if certificate verification were + /// turned off — the loopback tests all speak plain HTTP, and Warpgate does + /// disable it deliberately elsewhere (`warpgate-protocol-http`'s client + /// cache). The token crosses this connection, so it must not happen here. + #[tokio::test] + async fn test_an_untrusted_vault_certificate_is_refused() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let issued = rcgen::generate_simple_self_signed(vec!["localhost".to_owned()]).unwrap(); + let certificate = rustls_pki_types::CertificateDer::from(issued.cert.der().to_vec()); + let key = + rustls_pki_types::PrivateKeyDer::try_from(issued.signing_key.serialize_der()).unwrap(); + + let server_config = Arc::new( + rustls::ServerConfig::builder() + .with_no_client_auth() + .with_single_cert(vec![certificate], key) + .unwrap(), + ); + + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let acceptor = tokio_rustls::TlsAcceptor::from(server_config); + while let Ok((socket, _)) = listener.accept().await { + // The handshake is expected to fail; accepting is enough to + // prove the client got as far as looking at the certificate. + let _ = acceptor.accept(socket).await; + } + }); + + let secret_id_path = std::env::temp_dir().join("warpgate-vault-tls-test-secret"); + std::fs::write(&secret_id_path, "secret-id").unwrap(); + + let client = VaultClient::new(approle_config( + format!("https://localhost:{port}"), + secret_id_path, + )) + .unwrap(); + let error = client + .sign_ssh_key("warpgate", "ssh-ed25519 AAAA", "root", "warpgate:alice") + .await + .unwrap_err(); + + match error { + VaultError::Request(e) => { + let reported = format!("{e:?}"); + assert!( + reported.contains("certificate") || reported.contains("UnknownIssuer"), + "expected a certificate error, got {reported}" + ); + } + other => panic!("expected the handshake to be refused, got {other:?}"), + } + } + + #[test] + fn test_error_body_truncation_survives_a_split_character() { + // 255 ASCII bytes then a two-byte character: cutting at byte 256 lands + // inside it, which slicing a `String` would panic on. + let mut body = "a".repeat(255).into_bytes(); + body.extend_from_slice("é".as_bytes()); + + let rendered = render_error_body(&body[..MAX_ERROR_BODY], true); + assert!(rendered.starts_with(&"a".repeat(255))); + assert!(rendered.ends_with("... (truncated)")); + } + + #[test] + fn test_error_body_is_left_alone_when_it_fits() { + assert_eq!( + render_error_body(br#"{"errors":["permission denied"]}"#, false), + r#"{"errors":["permission denied"]}"# + ); + } + + #[test] + + fn test_address_validation() { + assert!(validate_address("https://vault.internal:8200").is_ok()); + assert!(validate_address("http://localhost:8200").is_ok()); + assert!(validate_address("http://127.0.0.1:8200").is_ok()); + assert!(matches!( + validate_address("http://vault.internal:8200"), + Err(VaultError::InsecureAddress) + )); + } + + #[test] + fn test_segment_validation() { + assert!(validate_segment("valid-role_123").is_ok()); + assert!(matches!( + validate_segment("../path-traversal"), + Err(VaultError::InvalidRole(_)) + )); + assert!(matches!( + validate_segment("role/with/slashes"), + Err(VaultError::InvalidRole(_)) + )); + } + + #[test] + fn test_principal_validation() { + assert!(validate_principal("root").is_ok()); + // Vault splits `valid_principals` on commas, so this would issue a + // certificate valid for root as well. + assert!(matches!( + validate_principal("deploy,root"), + Err(VaultError::InvalidPrincipal(_)) + )); + assert!(matches!( + validate_principal(""), + Err(VaultError::InvalidPrincipal(_)) + )); + assert!(matches!( + validate_principal("deploy\nroot"), + Err(VaultError::InvalidPrincipal(_)) + )); + } + + #[test] + fn test_key_id_validation() { + assert!(validate_key_id("warpgate:alice:6f1a").is_ok()); + // The target's sshd logs the key ID verbatim. + assert!(matches!( + validate_key_id("warpgate:alice\nAccepted publickey for root"), + Err(VaultError::InvalidKeyId) + )); + } + + #[test] + fn test_token_zeroizing() { + let secret = zeroize::Zeroizing::new("s.sensitive-vault-token".to_string()); + assert_eq!(secret.as_str(), "s.sensitive-vault-token"); + // Memory zeroized automatically on drop + } +} diff --git a/warpgate-vault/src/error.rs b/warpgate-vault/src/error.rs new file mode 100644 index 000000000..46eab3805 --- /dev/null +++ b/warpgate-vault/src/error.rs @@ -0,0 +1,77 @@ +use std::path::PathBuf; + +use thiserror::Error; + +pub type Result = std::result::Result; + +#[derive(Error, Debug)] +pub enum VaultError { + #[error("Vault request failed: {0}")] + Request(#[from] reqwest::Error), + + #[error("Vault returned {status}: {body}")] + Api { + status: reqwest::StatusCode, + body: String, + }, + + #[error("AWS: {0}")] + Aws(#[from] warpgate_aws::AwsError), + + #[error("serialization: {0}")] + Json(#[from] serde_json::Error), + + #[error("invalid metadata service address: {0}")] + MetadataAddress(#[from] url::ParseError), + + #[error("cannot read the Vault credential at {path}: {source}")] + CredentialFile { + path: PathBuf, + source: std::io::Error, + }, + + #[error("Vault address must use HTTPS for non-localhost endpoints")] + InsecureAddress, + + #[error("invalid Vault address: {0}")] + InvalidAddress(String), + + #[error("invalid Vault role or mount name: {0}")] + InvalidRole(String), + + #[error("invalid certificate principal: {0}")] + InvalidPrincipal(String), + + #[error("invalid certificate key ID")] + InvalidKeyId, + + #[error("Vault response is too large")] + OversizedResponse, +} + +impl VaultError { + pub fn client_message(&self) -> &'static str { + match self { + VaultError::InsecureAddress | VaultError::InvalidAddress(_) => { + "Vault endpoint configuration is invalid" + } + VaultError::InvalidRole(_) => "Invalid Vault role or mount configuration", + VaultError::InvalidPrincipal(_) | VaultError::InvalidKeyId => { + "Invalid certificate request parameters" + } + VaultError::Api { status, .. } => { + if status.is_client_error() { + "Vault denied the certificate signing request" + } else { + "Vault service error" + } + } + VaultError::CredentialFile { .. } => "Failed to read Vault credentials", + VaultError::Request(_) => "Vault is currently unavailable", + VaultError::Json(_) + | VaultError::MetadataAddress(_) + | VaultError::OversizedResponse => "Invalid response from Vault", + VaultError::Aws(e) => e.client_message(), + } + } +} diff --git a/warpgate-vault/src/lib.rs b/warpgate-vault/src/lib.rs new file mode 100644 index 000000000..3ad5cc965 --- /dev/null +++ b/warpgate-vault/src/lib.rs @@ -0,0 +1,6 @@ +mod client; +mod error; +mod metadata; + +pub use client::VaultClient; +pub use error::{Result, VaultError}; diff --git a/warpgate-vault/src/metadata.rs b/warpgate-vault/src/metadata.rs new file mode 100644 index 000000000..83c06de51 --- /dev/null +++ b/warpgate-vault/src/metadata.rs @@ -0,0 +1,94 @@ +use serde::Deserialize; +use url::Url; +use zeroize::Zeroizing; + +use crate::error::{Result, VaultError}; + +fn with_query(base: &str, path: &str, params: &[(&str, &str)]) -> Result { + let mut url = Url::parse(base)?.join(path)?; + url.query_pairs_mut().extend_pairs(params); + Ok(url) +} + +#[derive(Deserialize)] +struct AzureAccessToken { + access_token: String, +} + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct AzureInstance { + pub subscription_id: String, + pub resource_group_name: String, + pub name: String, + #[serde(default)] + pub vm_scale_set_name: String, +} + +/// The pieces Vault's Azure auth method needs: a token proving the VM's managed +/// identity, and the ARM coordinates it is checked against. +/// +/// `base` comes from the configuration, so an operator who can edit +/// `warpgate.yaml` can point these requests at a host of their choosing. That is +/// no more access than editing the config already grants, but it is why the +/// client refuses redirects. +pub async fn azure_login_material( + http: &reqwest::Client, + base: &str, + resource: &str, +) -> Result<(Zeroizing, AzureInstance)> { + let token: AzureAccessToken = http + .get(with_query( + base, + "/metadata/identity/oauth2/token", + &[("api-version", "2018-02-01"), ("resource", resource)], + )?) + .header("Metadata", "true") + .send() + .await? + .error_for_status() + .map_err(VaultError::Request)? + .json() + .await?; + + let instance: AzureInstance = http + .get(with_query( + base, + "/metadata/instance/compute", + &[("api-version", "2021-02-01")], + )?) + .header("Metadata", "true") + .send() + .await? + .error_for_status() + .map_err(VaultError::Request)? + .json() + .await?; + + Ok((Zeroizing::new(token.access_token), instance)) +} + +/// A GCE instance identity token, signed by Google for this specific audience. +/// `full` format includes the instance details Vault's `gce` auth type checks. +pub async fn gcp_identity_token( + http: &reqwest::Client, + base: &str, + audience: &str, +) -> Result> { + Ok(Zeroizing::new( + http.get(with_query( + base, + "/computeMetadata/v1/instance/service-accounts/default/identity", + &[("audience", audience), ("format", "full")], + )?) + .header("Metadata-Flavor", "Google") + .send() + .await? + .error_for_status() + .map_err(VaultError::Request)? + .text() + .await? + .trim() + .to_owned(), + )) +} diff --git a/warpgate-web/src/admin/config/targets/ssh/Options.svelte b/warpgate-web/src/admin/config/targets/ssh/Options.svelte index 79f76da8f..2fa975951 100644 --- a/warpgate-web/src/admin/config/targets/ssh/Options.svelte +++ b/warpgate-web/src/admin/config/targets/ssh/Options.svelte @@ -147,6 +147,9 @@ + + {/if} {#if options.auth.kind === 'Password'} , cli: &Cli) -> Result< let offset = UtcOffset::current_local_offset().unwrap_or(UtcOffset::UTC); - let env_filter = Arc::new(EnvFilter::from_default_env()); + // The AWS SDK logs the credentials it resolved — including the access key ID — + // at INFO. That is never worth having in Warpgate's log, so it is filtered out + // regardless of what RUST_LOG asks for; a target-specific directive outranks a + // broad one like `debug`. + let env_filter = + Arc::new(EnvFilter::from_default_env().add_directive("aws_config=warn".parse()?)); let enable_colors = console::user_attended(); // Determine effective log format (CLI overrides config) From bea50d46d1de7682f7813e723812210220bc62a7 Mon Sep 17 00:00:00 2001 From: Janis Dombrovskis Date: Tue, 11 Aug 2026 09:04:12 +0300 Subject: [PATCH 02/39] Close the vulnerabilities found in review of the certificate path - The admin host-key check ran on into authenticating to the target. On a certificate target that minted a real certificate and opened a real session nobody was attached to, held until the inactivity timeout, with a key ID naming no user. Now a dedicated RCCommand::CheckHostKey stops before authentication, on the final hop only so jump hosts still authenticate. - A certificate could arrive carrying critical options nobody asked for. A force-command there replaces what the user typed while keeping their own principal and key ID on the session, so the target's log attributes it to them. Write access to a Vault role is a lower bar than the right to sign with it, so this is the only place it can be caught. Refused by default; a target may name the options it expects and pin their values. - Nothing checked that the certificate named the account being reached. valid_principals is now verified against the target's username. - A response-wrapped AppRole secret ID was re-unwrapped on every login. A wrapping token is single-use, so every login after the first failed, as a generic denial. The unwrapped secret ID is now cached against the file content, and a genuine unwrap failure names the file and the fix. - lease_duration from Vault fed an unchecked Instant addition, so an oversized lease crashed the process on the login path. Now rejected as a bad response. - Cloud metadata tokens went through the same client as Vault, which honours HTTP_PROXY by default; GCE's hostname defeats a typical IP-based NO_PROXY. Metadata now uses a client built with no_proxy(). - The AWS login path was the one place credentials were not zeroized. - An IPv6 loopback Vault address was classified as a remote plaintext endpoint, because host_str renders it with brackets. - Editing the vault: section had no effect until a restart, alone among config sections. A VaultCell on a watch channel is rebuilt from run.rs; a configuration that fails to build keeps the working client. - A certificate Warpgate itself refused reported "SSH target rejected Warpgate's authentication request", naming the wrong party. It has its own error now, and the reason reaches the connecting user. - A role that forbids key IDs now produces a message naming allow_user_key_ids. Tests: 15 Rust unit and 57 integration, up from 12 and 48; each new one verified by breaking the code it defends. The stub models single-use wrapping tokens, without which the AppRole defect was invisible. Found by @theredspoon's review, which is worth more than the code it corrects. --- Cargo.lock | 1 + tests/stub_vault.py | 12 + tests/test_ssh_target_cert_auth.py | 281 +++++++++++++++++- warpgate-admin/src/api/ssh_connection_test.rs | 17 +- warpgate-common/src/config/mod.rs | 4 +- warpgate-common/src/config/target.rs | 19 ++ warpgate-core/src/lib.rs | 2 + warpgate-core/src/services.rs | 22 +- warpgate-core/src/vault_cell.rs | 34 +++ warpgate-protocol-http/src/api/info.rs | 4 +- warpgate-protocol-ssh/src/client/mod.rs | 193 +++++++++--- warpgate-vault/README.md | 36 ++- warpgate-vault/src/client.rs | 180 +++++++++-- warpgate-vault/src/error.rs | 23 +- .../admin/config/targets/ssh/Options.svelte | 61 ++++ .../src/admin/lib/openapi-schema.json | 27 +- .../src/gateway/lib/openapi-schema.json | 2 +- warpgate/Cargo.toml | 1 + warpgate/src/commands/run.rs | 31 ++ 19 files changed, 852 insertions(+), 98 deletions(-) create mode 100644 warpgate-core/src/vault_cell.rs diff --git a/Cargo.lock b/Cargo.lock index 44c0691ea..c7f1afb8f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9040,6 +9040,7 @@ dependencies = [ "warpgate-protocol-ssh", "warpgate-protocol-vnc", "warpgate-tls", + "warpgate-vault", ] [[package]] diff --git a/tests/stub_vault.py b/tests/stub_vault.py index 0d2008d8f..b0c5872db 100644 --- a/tests/stub_vault.py +++ b/tests/stub_vault.py @@ -215,6 +215,14 @@ def do_POST(self): if not token: self._reply(400, {"errors": ["missing wrapping token"]}) return + stub.unwraps.append(token) + # A wrapping token can be redeemed exactly once. Modelling that is + # the difference between a test that proves the secret ID survives + # more than one login and one that never asks. + if token in stub.spent_wrapping_tokens: + self._reply(400, {"errors": ["wrapping token is not valid or does not exist"]}) + return + stub.spent_wrapping_tokens.add(token) self._reply(200, {"data": {"secret_id": "unwrapped-secret-id"}}) return @@ -329,6 +337,8 @@ def __init__(self, directory: Path): self.signs = [] self.requests = [] self.metadata_requests = [] + self.unwraps = [] + self.spent_wrapping_tokens = set() self.valid_token = None self.reset() @@ -351,6 +361,8 @@ def reset(self): self.signs.clear() self.requests.clear() self.metadata_requests.clear() + self.unwraps.clear() + self.spent_wrapping_tokens.clear() def start(self): self._thread.start() diff --git a/tests/test_ssh_target_cert_auth.py b/tests/test_ssh_target_cert_auth.py index 93bc99462..8ae7621b2 100644 --- a/tests/test_ssh_target_cert_auth.py +++ b/tests/test_ssh_target_cert_auth.py @@ -9,11 +9,13 @@ """ import json +import time from pathlib import Path from uuid import uuid4 import psutil import pytest +import yaml from .api_client import admin_client, sdk from .conftest import ProcessManager, WarpgateProcess @@ -82,7 +84,15 @@ def reset_stub(stub_vault: StubVault): stub_vault.reset() -def make_user_and_target(api, ssh_port, *, role=None, username="root", assign=True): +def make_user_and_target( + api, + ssh_port, + *, + role=None, + username="root", + assign=True, + allowed_critical_options=None, +): wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) user = api.create_user(sdk.CreateUserRequest(username=f"user-{uuid4()}")) api.create_public_key_credential( @@ -105,7 +115,12 @@ def make_user_and_target(api, ssh_port, *, role=None, username="root", assign=Tr username=username, auth=sdk.SSHTargetAuth( sdk.SSHTargetAuthSshTargetCertificateAuth( - kind="Certificate", role=role + kind="Certificate", + role=role, + allowed_critical_options=[ + sdk.SshCertificateCriticalOption(name=name, value=value) + for name, value in (allowed_critical_options or []) + ], ) ), ) @@ -413,25 +428,92 @@ def test_a_success_that_carries_no_certificate( assert connect(processes, cert_wg, user, target, timeout)[0] != 0 assert len(stub_vault.signs) == 1 - def test_a_forced_command_from_the_issuer_is_reported( + def test_an_unexpected_forced_command_is_refused( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout ): """The target's sshd enforces critical options, so an issuer that - attaches `force-command` decides what the session runs — the user's own - command never executes, and the recording shows the output of something - they did not type. Warpgate asks for no critical options and cannot stop - the target honouring one, so the least it can do is say one arrived.""" - offset = Path(cert_wg.log_path).stat().st_size + attaches `force-command` decides what the session runs — under the + user's own principal and key ID, which makes the target's log attribute + the issuer's command to them. Planting one needs only write access to a + Vault role, not the right to sign or a route to the target, so Warpgate + is the only place this can be caught.""" stub_vault.sign_options = ["force-command=echo chosen-by-the-issuer"] user, target = make_user_and_target(api, cert_ssh_port) code, stdout = connect(processes, cert_wg, user, target, timeout) - # Not an assumption: this is the mechanism the test exists for. - assert (code, stdout) == (0, b"chosen-by-the-issuer\n") + assert code != 0 + assert b"chosen-by-the-issuer" not in stdout, "the forced command ran" + assert stub_vault.signs, "no certificate was issued, so nothing was refused" + + def test_the_refusal_reaches_the_user_not_only_the_log( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A server-side warning nobody is watching is not a control. Whoever is + connecting has to be told, and told that it was Warpgate that refused — + "the target rejected you" sends them to the wrong machine.""" + stub_vault.sign_options = ["force-command=echo chosen-by-the-issuer"] + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + stdout = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert "Warpgate refused the certificate" in stdout + assert "force-command" in stdout + + def test_a_critical_option_the_target_expects_is_allowed_through( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A restricted Vault role may set `default_critical_options` on + purpose. Naming it on the target is how an operator says so.""" + stub_vault.sign_options = ["force-command=echo expected-by-the-operator"] + user, target = make_user_and_target( + api, + cert_ssh_port, + allowed_critical_options=[ + ("force-command", "echo expected-by-the-operator") + ], + ) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert (code, stdout) == (0, b"expected-by-the-operator\n") + + def test_a_pinned_value_must_match( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Allowing the name alone would let the issuer choose the command; the + point of pinning is that the value is the part that matters.""" + stub_vault.sign_options = ["force-command=echo something-else-entirely"] + user, target = make_user_and_target( + api, + cert_ssh_port, + allowed_critical_options=[("force-command", "echo the-expected-one")], + ) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert code != 0 + assert b"something-else-entirely" not in stdout + + def test_a_certificate_naming_the_wrong_account_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Vault returns the requested principals verbatim or refuses, so a set + that omits the account being reached did not come from this request. + + The target's sshd would refuse this certificate too, which is exactly + why the assertion is on who did the refusing: without Warpgate's own + check the session still fails, just with the wrong explanation and + after the certificate has been put on the wire. + """ + stub_vault.principals = "someone-else" + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + stdout = client.communicate(timeout=timeout)[0].decode(errors="replace") - log = log_since(cert_wg, offset) - assert "critical options" in log - assert "force-command" in log + assert client.returncode != 0 + assert stub_vault.signs, "no certificate was issued, so nothing was refused" + assert "Warpgate refused the certificate" in stdout + assert "does not name the target account root" in stdout class TestIssuerFailures: @@ -841,6 +923,179 @@ def test_approle_response_wrapping( assert login["method"] == "approle" assert login["secret_id"] == "unwrapped-secret-id" + def test_a_wrapping_token_is_redeemed_once_and_the_secret_id_reused( + self, processes, cert_ssh_port, stub_vault, ctx, timeout + ): + """A wrapping token can be redeemed exactly once, while the secret ID + inside it stays usable. Unwrapping per login would leave every session + after the first unable to authenticate to Vault at all.""" + secret_id_path = ctx.tmpdir / f"wrapping-token-{uuid4()}" + secret_id_path.write_text("unwrap:stub-wrapping-token") + + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "app_role", + "role_id": "role-1", + "secret_id_path": str(secret_id_path), + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + # Forces a second login rather than waiting out the lease. + stub_vault.invalidate_token() + assert connect(processes, wg, user, target, timeout)[0] == 0 + + assert len(stub_vault.logins) == 2, "the second session did not log in again" + assert stub_vault.logins[-1]["secret_id"] == "unwrapped-secret-id" + assert len(stub_vault.unwraps) == 1, "the wrapping token was redeemed twice" + + def test_a_freshly_provisioned_wrapping_token_is_picked_up( + self, processes, cert_ssh_port, stub_vault, ctx, timeout + ): + """Caching the unwrapped secret ID must not mean ignoring the file: an + operator rotating the credential writes a new wrapping token there.""" + secret_id_path = ctx.tmpdir / f"wrapping-token-{uuid4()}" + secret_id_path.write_text("unwrap:first-wrapping-token") + + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "app_role", + "role_id": "role-1", + "secret_id_path": str(secret_id_path), + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + secret_id_path.write_text("unwrap:second-wrapping-token") + stub_vault.invalidate_token() + assert connect(processes, wg, user, target, timeout)[0] == 0 + + assert stub_vault.unwraps == ["first-wrapping-token", "second-wrapping-token"] + + +class TestConfigReload: + def test_a_new_vault_address_takes_effect_without_a_restart( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """Every other section of the config is watched and applied live. A + `vault:` that quietly needed a restart would be the one exception, and + an operator editing it would have no way of knowing.""" + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + second = StubVault(ctx.tmpdir / f"stub-vault-reload-{uuid4()}") + second.start() + try: + port = processes.start_ssh_server(trusted_ca=[second.ca_public_key]) + wait_port(port) + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, port) + + # The running config points at the first issuer, whose CA this + # target does not trust — so this must fail before the edit. + assert connect(processes, wg, user, target, timeout)[0] != 0 + + config = yaml.safe_load(wg.config_path.read_text()) + config["vault"]["address"] = second.url + wg.config_path.write_text(yaml.dump(config)) + + # The watcher debounces for 500ms before reloading once. + deadline = time.time() + 20 + while time.time() < deadline: + if second.signs: + break + connect(processes, wg, user, target, timeout) + time.sleep(1) + + assert second.signs, "the edited Vault address was never picked up" + assert connect(processes, wg, user, target, timeout)[0] == 0 + finally: + second.stop() + + +class TestTheHostKeyCheck: + """The admin host-key check reaches the same connection code the SSH path + does. It asks for one thing — the target's host key — and must not carry on + into authentication behind the operator's back.""" + + def test_checking_a_host_key_issues_no_certificate( + self, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Pressing the button on a certificate target would otherwise mint a + real certificate and open a real authenticated session that nothing is + attached to, holding until the inactivity timeout.""" + _, target = make_user_and_target(api, cert_ssh_port) + + first = api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + assert first.remote_key_base64, "the check did not reach the target at all" + + # The first press leaves the key trusted; it is every press after that + # one which used to run on into authentication. + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + + # The request returns as soon as the key arrives; a connection that kept + # going would reach the issuer a moment afterwards, so the assertion has + # to outlast the response rather than race it. + deadline = time.time() + 5 + while time.time() < deadline: + assert stub_vault.signs == [], "the host key check issued a certificate" + time.sleep(0.25) + + def test_the_check_leaves_no_session_behind( + self, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A session opened by the check is invisible in the UI and outlives the + request, so counting what the gateway is still holding is the only way + to see it.""" + _, target = make_user_and_target(api, cert_ssh_port) + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + + # Measured as a delta: the gateway is shared by the whole module, so + # earlier tests have left sessions and sockets of their own behind. + gateway = psutil.Process(cert_wg.process.pid) + target_socket_count = lambda: len( + [c for c in gateway.net_connections(kind="tcp") if c.raddr and c.raddr.port == cert_ssh_port] + ) + before_sockets = target_socket_count() + before_sessions = api.get_sessions().total + + for _ in range(3): + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + + assert api.get_sessions().total == before_sessions, "the check registered a session" + # A connection still open to the target is the observable trace of a + # client task that never exited. One-sided on purpose: an earlier test's + # socket finishing its close during this window lowers the count, and + # that is not what is under test. + assert target_socket_count() <= before_sockets class TestTheStubItself: diff --git a/warpgate-admin/src/api/ssh_connection_test.rs b/warpgate-admin/src/api/ssh_connection_test.rs index 99643e69f..cb1c24690 100644 --- a/warpgate-admin/src/api/ssh_connection_test.rs +++ b/warpgate-admin/src/api/ssh_connection_test.rs @@ -49,9 +49,19 @@ impl Api { .collect::>(); let mut handles = RemoteClient::create(Uuid::new_v4(), admin.services().clone())?; + // Not `Connect`: that would carry on into authenticating to the target + // once the key had been reported, opening a session nothing is attached + // to — and, for a certificate target, minting a real certificate to do + // it with. let _ = handles .command_tx - .send((RCCommand::Connect(ssh_chain), None)); + .send((RCCommand::CheckHostKey(ssh_chain), None)); + + // Kept out of the future below so the connection can be torn down after + // it resolves. `CheckHostKey` already ends the client task on its own; + // this is the second lock on the same door, scoped to this caller so + // that a real session's graceful disconnect stays untouched. + let abort_tx = handles.abort_tx.clone(); let fut = async move { let key = loop { @@ -70,9 +80,12 @@ impl Api { anyhow::Ok(key) }; + let result = fut.await; + let _ = abort_tx.send(()); + // Result is matched manually since we need to manually format // the error message with :# to included the nested errors here - match fut.await { + match result { Ok(key) => Ok(CheckSshHostKeyResponse::Ok(Json( CheckSshHostKeyResponseBody { remote_key_type: key.algorithm().as_str().into(), diff --git a/warpgate-common/src/config/mod.rs b/warpgate-common/src/config/mod.rs index bdbeac481..43db295da 100644 --- a/warpgate-common/src/config/mod.rs +++ b/warpgate-common/src/config/mod.rs @@ -420,7 +420,7 @@ pub enum LogFormat { /// config would put it straight back. A Kubernetes service account token is /// mounted and rotated by the kubelet; an AppRole secret ID is short-lived and /// meant to be delivered response-wrapped by whatever provisions the host. -#[derive(Debug, Deserialize, Serialize, Clone, JsonSchema)] +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, JsonSchema)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum VaultAuth { Kubernetes { @@ -475,7 +475,7 @@ impl VaultAuth { } } -#[derive(Debug, Deserialize, Serialize, Clone, JsonSchema)] +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, JsonSchema)] pub struct VaultConfig { /// Base URL of the Vault server, e.g. `https://vault.internal:8200`. pub address: String, diff --git a/warpgate-common/src/config/target.rs b/warpgate-common/src/config/target.rs index 5986820f3..474050671 100644 --- a/warpgate-common/src/config/target.rs +++ b/warpgate-common/src/config/target.rs @@ -77,6 +77,25 @@ pub struct SshTargetCertificateAuth { /// targets of differing privilege belong to differing roles. #[serde(default)] pub role: Option, + + /// Critical options this target's certificates are expected to carry. + /// Anything not listed here is refused, because the target's sshd enforces + /// whatever arrives: a `force-command` decides what the session runs, under + /// the connecting user's own principal and key ID. + #[serde(default)] + #[oai(default)] + pub allowed_critical_options: Vec, +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Object)] +pub struct SshCertificateCriticalOption { + /// Option name as it appears in the certificate, e.g. `force-command`. + pub name: String, + + /// The exact value required. Unset accepts any value for this name — worth + /// avoiding for `force-command`, whose value is the command that runs. + #[serde(default)] + pub value: Option, } #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Object, Default)] diff --git a/warpgate-core/src/lib.rs b/warpgate-core/src/lib.rs index a6486d37e..3b18c8da8 100644 --- a/warpgate-core/src/lib.rs +++ b/warpgate-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod recordings; mod services; mod state; pub mod ticket_requests; +mod vault_cell; pub use auth_state_store::*; pub use config_providers::*; pub use credential_encryption::*; @@ -28,3 +29,4 @@ pub use listener_status::*; pub use protocols::*; pub use services::*; pub use state::{SessionState, SessionStateInit, State}; +pub use vault_cell::VaultCell; diff --git a/warpgate-core/src/services.rs b/warpgate-core/src/services.rs index 615e0cad8..915a1be8d 100644 --- a/warpgate-core/src/services.rs +++ b/warpgate-core/src/services.rs @@ -19,6 +19,7 @@ use crate::rate_limiting::RateLimiterRegistry; use crate::recordings::SessionRecordings; use crate::{ AuthStateStore, ConfigProviderEnum, DatabaseConfigProvider, ListenerStatusRegistry, State, + VaultCell, }; #[derive(Clone)] @@ -29,8 +30,9 @@ pub struct Services { pub cluster: Arc, pub state: Arc>, pub config_provider: Arc, - /// Present only when the config declares a Vault server. - pub vault: Option>, + /// Empty unless the config declares a Vault server. Swappable so that + /// editing `vault:` takes effect without a restart. + pub vault: VaultCell, pub auth_state_store: Arc>, pub admin_token: Arc>>, pub cluster_token: Arc>, @@ -77,13 +79,15 @@ impl Services { let cluster = Arc::new(Cluster::new(db.clone(), config.store.http.listen.port()).await?); - let vault = config - .store - .vault - .clone() - .map(VaultClient::new) - .transpose()? - .map(Arc::new); + let vault = VaultCell::new( + config + .store + .vault + .clone() + .map(VaultClient::new) + .transpose()? + .map(Arc::new), + ); let config = Arc::new(Mutex::new(config)); diff --git a/warpgate-core/src/vault_cell.rs b/warpgate-core/src/vault_cell.rs new file mode 100644 index 000000000..9fae8633d --- /dev/null +++ b/warpgate-core/src/vault_cell.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; + +use tokio::sync::watch; +use warpgate_vault::VaultClient; + +/// Houses a replaceable reference to the Vault client, so that editing the +/// `vault:` section takes effect the way editing any other section does. +/// +/// Built on a `watch` channel for the same reason the listener supervisors are: +/// a client is not something a session can hold across a reload, since the +/// address, the mount or the way Warpgate proves its identity may all have +/// changed. Readers take a fresh reference per use; each keeps working until it +/// is dropped, so a session already in flight is never cut short by a reload. +#[derive(Clone)] +pub struct VaultCell { + receiver: watch::Receiver>>, + sender: watch::Sender>>, +} + +impl VaultCell { + pub fn new(client: Option>) -> Self { + let (sender, receiver) = watch::channel(client); + Self { receiver, sender } + } + + /// The client as of right now, or `None` when no Vault server is configured. + pub fn get(&self) -> Option> { + self.receiver.borrow().clone() + } + + pub fn replace(&self, client: Option>) { + let _ = self.sender.send(client); + } +} diff --git a/warpgate-protocol-http/src/api/info.rs b/warpgate-protocol-http/src/api/info.rs index fdb2e3590..b62e184a6 100644 --- a/warpgate-protocol-http/src/api/info.rs +++ b/warpgate-protocol-http/src/api/info.rs @@ -358,7 +358,9 @@ impl Api { } else { None }, - has_vault: auth_ctx.is_some().then(|| ctx.services().vault.is_some()), + has_vault: auth_ctx + .is_some() + .then(|| ctx.services().vault.get().is_some()), should_prompt_analytics, banner: parameters.banner.clone(), show_session_menu: parameters.show_session_menu, diff --git a/warpgate-protocol-ssh/src/client/mod.rs b/warpgate-protocol-ssh/src/client/mod.rs index 236671f5c..fa16af5e4 100644 --- a/warpgate-protocol-ssh/src/client/mod.rs +++ b/warpgate-protocol-ssh/src/client/mod.rs @@ -30,7 +30,10 @@ use tracing::*; use uuid::Uuid; use warpgate_aws::AwsError; use warpgate_common::helpers::rng::get_crypto_rng; -use warpgate_common::{SSHTargetAuth, SessionId, TargetOptions, TargetSSHOptions, WarpgateError}; +use warpgate_common::{ + SSHTargetAuth, SessionId, SshCertificateCriticalOption, TargetOptions, TargetSSHOptions, + WarpgateError, +}; use warpgate_core::{ConfigProvider, Services}; use self::handler::ClientHandlerEvent; @@ -75,6 +78,12 @@ pub enum ConnectionError { #[error("Authentication failed")] Authentication, + /// Warpgate refused the credential before offering it, so the target never + /// saw anything — saying it was "rejected by the target" would name the + /// wrong party and send the operator to the wrong logs. + #[error("Certificate refused by Warpgate: {0}")] + CertificateRefused(String), + #[error("Jump host target not found")] JumpHostTargetNotFound, @@ -90,6 +99,9 @@ impl ConnectionError { ConnectionError::Authentication => { "SSH target rejected Warpgate's authentication request".to_string() } + ConnectionError::CertificateRefused(reason) => { + format!("Warpgate refused the certificate issued for this session: {reason}") + } ConnectionError::HostKeyMismatch { .. } => "Host key mismatch".to_string(), ConnectionError::Resolve => "Could not resolve target address".to_string(), ConnectionError::Aborted => "Connection aborted".to_string(), @@ -104,16 +116,60 @@ impl ConnectionError { } /// Vault is a second gate, not a trusted party, so what comes back is checked -/// against what was asked for before it is offered to the target. Neither of -/// these could authenticate anything anyway — the point is to say so in the -/// session log rather than let the target's refusal stand in for the reason. -fn certificate_mismatch(certificate: &Certificate, key: &PublicKey) -> Option<&'static str> { +/// against what was asked for before it is offered to the target. +/// +/// The type and key checks are belt-and-braces — neither could authenticate +/// anything — but the last two are load-bearing. The target's sshd enforces +/// whatever critical options arrive, so a `force-command` planted on a role +/// replaces what the user asked to run while keeping their principal and key ID +/// on it: the target's own log then attributes the attacker's command to them. +/// Writing a role is a lower bar than signing with it, so this is the only place +/// the check can land. +fn certificate_mismatch( + certificate: &Certificate, + key: &PublicKey, + principal: &str, + allowed_options: &[SshCertificateCriticalOption], +) -> Option { if certificate.cert_type() != CertType::User { - return Some("Vault returned a host certificate rather than a user certificate"); + return Some("Vault returned a host certificate rather than a user certificate".to_owned()); } if certificate.public_key() != key.key_data() { - return Some("Vault signed a key other than the one generated for this session"); + return Some("Vault signed a key other than the one generated for this session".to_owned()); + } + // Vault returns the requested set verbatim — trimmed, deduped and sorted — + // or refuses outright, so a set that no longer contains the account being + // reached means the answer did not come from the request that was made. + if !certificate + .valid_principals() + .iter() + .any(|candidate| candidate == principal) + { + return Some(format!( + "Vault issued a certificate that does not name the target account {principal}" + )); + } + + for (name, value) in certificate.critical_options().iter() { + let permitted = allowed_options.iter().find(|option| &option.name == name); + match permitted { + None => { + return Some(format!( + "Vault issued a certificate carrying the critical option {name}, which this target does not allow" + )); + } + Some(option) => { + if let Some(expected) = &option.value + && expected != value + { + return Some(format!( + "Vault issued a certificate whose critical option {name} does not match the value configured for this target" + )); + } + } + } } + None } @@ -278,6 +334,14 @@ pub type RCCommandReply = oneshot::Sender>; #[derive(Clone, Debug)] pub enum RCCommand { Connect(Vec), + /// Connect only as far as the target's host key, then stop. + /// + /// A separate command rather than the caller dropping its handle once it has + /// what it wants: the connection otherwise runs on into authentication, and + /// for a certificate target that means a real certificate issued and a real + /// session opened for nobody. Cancelling on a dropped handle is a race the + /// caller loses about half the time; refusing to start is not. + CheckHostKey(Vec), Channel(Uuid, ChannelOperation), ForwardTCPIP(String, u32), CancelTCPIPForward(String, u32), @@ -578,6 +642,14 @@ impl RemoteClient { return Ok(true); } }, + RCCommand::CheckHostKey(options) => { + if let Err(e) = self.check_host_key(options).await { + debug!("Host key check error: {}", e); + let _ = self.tx.send(RCEvent::ConnectionError(e)).await; + } + self.set_disconnected().await; + return Ok(true); + } RCCommand::Channel(ch, op) => { self.apply_channel_op(ch, op).await?; } @@ -687,11 +759,18 @@ impl RemoteClient { /// Connect through a pre-resolved chain of SSH hops, each tunnelled through the previous. /// `chain` must be non-empty; the first entry is connected directly, subsequent ones via /// `channel_open_direct_tcpip` through the previous session. + /// `stop_after_host_key` applies to the final hop only: the intermediate + /// ones must authenticate, or there is no tunnel to carry the last one. + /// Returns `None` when it stopped at the host key as asked. async fn connect_chain( &mut self, chain: Vec, - ) -> Result<(Handle, UnboundedReceiver), ConnectionError> - { + stop_after_host_key: bool, + ) -> Result< + Option<(Handle, UnboundedReceiver)>, + ConnectionError, + > { + let hop_count = chain.len(); let mut iter = chain.into_iter(); let first = iter.next().ok_or(ConnectionError::Resolve)?; @@ -711,12 +790,16 @@ impl RemoteClient { session_id: self.id, }; let fut = russh::client::connect(config, address, handler); - let (mut session, mut active_rx) = self - .wait_for_connection(&first, fut, event_rx, false) + let Some((mut session, mut active_rx)) = self + .wait_for_connection(&first, fut, event_rx, stop_after_host_key && hop_count == 1) .boxed() - .await?; + .await? + else { + return Ok(None); + }; - for ssh_options in iter { + for (index, ssh_options) in iter.enumerate() { + let is_last = index + 2 == hop_count; let _ = self.tx.send(RCEvent::HopConnected).await; info!( host = %ssh_options.host, @@ -742,19 +825,42 @@ impl RemoteClient { session_id: self.id, }; let fut = russh::client::connect_stream(config, stream, handler); - let (new_session, new_rx) = self - .wait_for_connection(&ssh_options, fut, event_rx, false) + let Some((new_session, new_rx)) = self + .wait_for_connection(&ssh_options, fut, event_rx, stop_after_host_key && is_last) .boxed() - .await?; + .await? + else { + return Ok(None); + }; session = new_session; active_rx = new_rx; } - Ok((session, active_rx)) + Ok(Some((session, active_rx))) + } + + /// Connects as far as the target's host key and stops, for the admin-side + /// check. Nothing is authenticated, so nothing is issued. + async fn check_host_key( + &mut self, + chain: Vec, + ) -> Result<(), ConnectionError> { + if let Some((session, _)) = self.connect_chain(chain, true).boxed().await? { + // Only reachable if the connection came up without the handler ever + // reporting a host key, which should not happen — but an open + // session left behind would be exactly the leak this command exists + // to close. + let _ = session + .disconnect(russh::Disconnect::ByApplication, "", "") + .await; + } + Ok(()) } async fn connect(&mut self, chain: Vec) -> Result<(), ConnectionError> { - let (session, mut event_rx) = self.connect_chain(chain).boxed().await?; + let Some((session, mut event_rx)) = self.connect_chain(chain, false).boxed().await? else { + return Err(ConnectionError::Internal); + }; self.session = Some(Arc::new(Mutex::new(session))); @@ -782,8 +888,11 @@ impl RemoteClient { ssh_options: &TargetSSHOptions, fut_connect: Fut, mut event_rx: UnboundedReceiver, - _is_jump_host: bool, - ) -> Result<(Handle, UnboundedReceiver), ConnectionError> + stop_after_host_key: bool, + ) -> Result< + Option<(Handle, UnboundedReceiver)>, + ConnectionError, + > where Fut: Future, ClientHandlerError>>, { @@ -795,14 +904,24 @@ impl RemoteClient { match event { ClientHandlerEvent::HostKeyReceived(key) => { self.tx.send(RCEvent::HostKeyReceived(key)).await.map_err(|_| ConnectionError::Internal)?; + if stop_after_host_key { + return Ok(None); + } } ClientHandlerEvent::HostKeyUnknown(key, reply) => { self.tx.send(RCEvent::HostKeyUnknown(key, reply)).await.map_err(|_| ConnectionError::Internal)?; + if stop_after_host_key { + return Ok(None); + } } _ => {} } } - Some(()) = self.abort_rx.recv() => { + // `None` means every sender is gone, so the owner has dropped + // without disconnecting. `ServerSession::drop` signals first, so + // a live session never arrives here still wanting the + // connection — anything else that does is abandoning it. + _ = self.abort_rx.recv() => { info!("Abort requested"); self.set_disconnected().await; return Err(ConnectionError::Aborted) @@ -829,7 +948,7 @@ impl RemoteClient { ssh_options.allow_insecure_algos.unwrap_or(false) ).await?; - return Ok((session, event_rx)); + return Ok(Some((session, event_rx))); } } } @@ -945,7 +1064,7 @@ impl RemoteClient { } } SSHTargetAuth::Certificate(auth) => { - if let Some(vault) = self.services.vault.clone() { + if let Some(vault) = self.services.vault.get() { // The key exists only for this authentication attempt — nothing // the target will ever trust again outlives this scope. let key = Arc::new( @@ -965,21 +1084,27 @@ impl RemoteClient { let certificate = Certificate::from_openssh(&signed_key).map_err(russh::keys::Error::from)?; - // The target's sshd enforces critical options, so one of - // these decides what the session actually does — a forced - // command runs instead of whatever the user asked for. - // Warpgate never requests any, so an operator has to be able - // to see that one arrived. - if !certificate.critical_options().is_empty() { - warn!( - options = ?certificate.critical_options().keys().collect::>(), + // Extensions do not change what runs, but an unexpected one + // still says the role is not what the operator thinks. + if !certificate.extensions().is_empty() { + debug!( + extensions = ?certificate.extensions().keys().collect::>(), key_id, - "Vault issued a certificate carrying critical options" + "Certificate carries extensions" ); } - if let Some(reason) = certificate_mismatch(&certificate, key.public_key()) { - auth_error_msg = Some(reason.into()); + if let Some(reason) = certificate_mismatch( + &certificate, + key.public_key(), + username, + &auth.allowed_critical_options, + ) { + // Surfaced to the user, not only to the log: a session + // that dies with "the target rejected you" sends whoever + // is debugging it to the wrong machine entirely. + warn!(key_id, reason, "Refusing the issued certificate"); + return Err(ConnectionError::CertificateRefused(reason)); } else { let response = session .authenticate_openssh_cert(username.to_string(), key, certificate) diff --git a/warpgate-vault/README.md b/warpgate-vault/README.md index e51900c6a..3d583f608 100644 --- a/warpgate-vault/README.md +++ b/warpgate-vault/README.md @@ -68,6 +68,11 @@ Two settings deserve attention: - **`allow_user_key_ids`** must be `true`. Without it Vault rejects the request outright, and every session would be attributed to the Warpgate service identity instead of to a person. +- **A target with no username** asks for a certificate naming the *connecting + Warpgate user* instead — the same substitution every other SSH auth method + makes. That is not a bypass, since `allowed_users` still bounds what Vault will + issue, but it means the role must list every Warpgate username that may reach + the target. Setting the username explicitly is almost always what you want. `default_extensions` grants the minimum. Add `permit-port-forwarding` or `permit-agent-forwarding` only if your users need them. @@ -320,17 +325,32 @@ token cannot send every session in flight to Vault at once. The cost is that a login which hangs blocks the others for up to `timeout`. Keep `timeout` at a value you are willing to make every concurrent session wait. -**A certificate can come back carrying critical options.** If a Vault role sets -`default_critical_options`, a `force-command` there replaces whatever the user -typed — the target runs Vault's command and the session recording shows its -output. Warpgate does not refuse such a certificate, because a restricted role -may set one deliberately, but it logs every arrival: +**Critical options are refused unless the target names them.** If a Vault role +sets `default_critical_options`, a `force-command` there replaces whatever the +user typed — the target runs Vault's command, under the user's own principal and +key ID, and the recording shows its output. Planting one needs write access to a +role, which is a lower bar than the right to sign with it or a route to the +target, so Warpgate is the only place this can be caught. +A target therefore refuses any critical option it was not told to expect. Where a +role sets one deliberately, name it on the target — and pin the value, since for +`force-command` the value is the command: + +```json +"auth": { + "kind": "Certificate", + "role": "warpgate-backup", + "allowed_critical_options": [ + { "name": "force-command", "value": "/usr/local/bin/backup" } + ] +} ``` -WARN Vault issued a certificate carrying critical options options=["force-command"] -``` -Alert on that line if your roles are not supposed to set any. +Leaving `value` unset accepts any value for that name. The refusal reaches the +connecting user, not only the log. + +Extensions are not refused — they cannot change what runs — but an unexpected one +is logged at debug level, since it still says the role is not what you think. **`address` and `metadata_address` are fetched as given.** Both come from `warpgate.yaml`, so anyone who can edit that file can point Warpgate's outbound diff --git a/warpgate-vault/src/client.rs b/warpgate-vault/src/client.rs index 75678447f..696ead3eb 100644 --- a/warpgate-vault/src/client.rs +++ b/warpgate-vault/src/client.rs @@ -34,9 +34,16 @@ static NEXT_TOKEN_ID: AtomicU64 = AtomicU64::new(1); fn validate_address(address: &str) -> Result<()> { let parsed = url::Url::parse(address).map_err(|e| VaultError::InvalidAddress(e.to_string()))?; if parsed.scheme() != "https" { - let host = parsed.host_str().unwrap_or_default(); - let is_localhost = host == "localhost" || host == "127.0.0.1" || host == "::1"; - if !is_localhost { + // Matched on the parsed host rather than on `host_str`, which renders an + // IPv6 address with its brackets — `[::1]` never equalled `::1`, so a + // loopback Vault over IPv6 was refused as though it were remote. + let is_loopback = match parsed.host() { + Some(url::Host::Domain(domain)) => domain == "localhost", + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + None => false, + }; + if !is_loopback { return Err(VaultError::InsecureAddress); } } @@ -92,6 +99,19 @@ struct CachedToken { expires_at: Option, } +/// A secret ID that was delivered response-wrapped, kept alongside the exact +/// file content it came from. +/// +/// A wrapping token can be redeemed once, while the secret ID inside it stays +/// usable until its own `secret_id_num_uses` or `secret_id_ttl` runs out — so +/// unwrapping per login would fail every login after the first. Keying on the +/// file content means a freshly provisioned token is still picked up: the file +/// is read on every login, only the redemption is skipped. +struct UnwrappedSecretId { + source: Zeroizing, + secret_id: Zeroizing, +} + fn login_payload(value: &T) -> Result> { Ok(Zeroizing::new(serde_json::to_string(value)?)) } @@ -123,9 +143,9 @@ struct AzureLogin<'a> { struct AwsLogin<'a> { role: Option<&'a str>, iam_http_request_method: &'a str, - iam_request_url: String, - iam_request_body: String, - iam_request_headers: String, + iam_request_url: &'a str, + iam_request_body: &'a str, + iam_request_headers: &'a str, } #[derive(Serialize)] @@ -174,7 +194,9 @@ struct UnwrapData { pub struct VaultClient { config: VaultConfig, http: reqwest::Client, + metadata_http: reqwest::Client, token: Mutex>, + unwrapped_secret_id: Mutex>, } impl VaultClient { @@ -197,16 +219,30 @@ impl VaultClient { // `Authorization` on a cross-origin hop but knows nothing about // `X-Vault-Token`, so a 307 from a hostile or misconfigured endpoint // would replay the token to another host, or downgrade the request to - // plain HTTP. The same client fetches cloud metadata, where following a - // redirect would be just as wrong. + // plain HTTP. let http = reqwest::Client::builder() .redirect(reqwest::redirect::Policy::none()) .timeout(config.timeout) .build()?; + + // The metadata services are link-local and plain HTTP by definition, and + // reqwest honours HTTP_PROXY/HTTPS_PROXY by default — a proxy in the + // environment would carry the instance identity token off the host. + // GCE's default address is a hostname, so the usual IP-based NO_PROXY + // list does not cover it. Vault's own address keeps ambient proxy + // support, since reaching it through one is a legitimate deployment. + let metadata_http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .timeout(config.timeout) + .build()?; + Ok(Self { config, http, + metadata_http, token: Mutex::new(None), + unwrapped_secret_id: Mutex::new(None), }) } @@ -345,10 +381,20 @@ impl VaultClient { // Vault reports a lease of zero for a token that does not expire. // Reading that as "expired half a minute ago" would turn every // certificate request into a fresh login. - expires_at: (auth.lease_duration > 0).then(|| { - Instant::now() - + Duration::from_secs(auth.lease_duration).saturating_sub(TOKEN_EXPIRY_MARGIN) - }), + // + // The lease is an unbounded number out of a network response, and + // `Instant + Duration` panics on overflow, so an absurd one is + // rejected as a bad response rather than taken at its word. + expires_at: match auth.lease_duration { + 0 => None, + seconds => Some( + Instant::now() + .checked_add( + Duration::from_secs(seconds).saturating_sub(TOKEN_EXPIRY_MARGIN), + ) + .ok_or(VaultError::InvalidLease(seconds))?, + ), + }, }) } @@ -374,9 +420,7 @@ impl VaultClient { } => { let cred = Self::read_credential(secret_id_path).await?; let secret_id = if cred.starts_with("unwrap:") { - let wrapping_token = - Zeroizing::new(cred.trim_start_matches("unwrap:").trim().to_owned()); - self.unwrap_secret_id(&wrapping_token).await? + self.unwrapped_secret_id(secret_id_path, &cred).await? } else { cred }; @@ -403,7 +447,8 @@ impl VaultClient { metadata_address, } => { let (jwt, instance) = - metadata::azure_login_material(&self.http, metadata_address, resource).await?; + metadata::azure_login_material(&self.metadata_http, metadata_address, resource) + .await?; ( "auth/azure/login", login_payload(&AzureLogin { @@ -423,7 +468,8 @@ impl VaultClient { } => { let audience = format!("vault/{role}"); let jwt = - metadata::gcp_identity_token(&self.http, metadata_address, &audience).await?; + metadata::gcp_identity_token(&self.metadata_http, metadata_address, &audience) + .await?; ( "auth/gcp/login", login_payload(&JwtLogin { role, jwt: &jwt })?, @@ -432,6 +478,36 @@ impl VaultClient { }) } + /// The secret ID behind a `unwrap:` file, redeeming the wrapping + /// token the first time and on every later change to the file. + async fn unwrapped_secret_id( + &self, + path: &Path, + cred: &Zeroizing, + ) -> Result> { + let mut cached = self.unwrapped_secret_id.lock().await; + + if let Some(entry) = cached.as_ref() + && entry.source.as_str() == cred.as_str() + { + return Ok(entry.secret_id.clone()); + } + + let wrapping_token = Zeroizing::new(cred.trim_start_matches("unwrap:").trim().to_owned()); + let secret_id = self.unwrap_secret_id(&wrapping_token).await.map_err(|e| { + VaultError::SecretIdUnwrap { + path: path.to_owned(), + source: Box::new(e), + } + })?; + + *cached = Some(UnwrappedSecretId { + source: cred.clone(), + secret_id: secret_id.clone(), + }); + Ok(secret_id) + } + async fn unwrap_secret_id(&self, wrapping_token: &str) -> Result> { let response = self .http @@ -464,18 +540,27 @@ impl VaultClient { server_id: Option<&str>, region: Option<&str>, ) -> Result> { - let request = warpgate_aws::sign_sts_identity_request(region, server_id).await?; - // Carries the signature and, on an instance role, the session token. - let mut headers = serde_json::to_string(&request.headers)?; + let mut request = warpgate_aws::sign_sts_identity_request(region, server_id).await?; + + // The headers carry the SigV4 signature and, on an instance role, the + // session token — the one part of this request worth protecting. The URL + // and body are the same public constants on every call. Every buffer + // they pass through is zeroized, to match what the other methods do with + // their credentials. + let headers = Zeroizing::new(serde_json::to_string(&request.headers)?); + let encoded_headers = Zeroizing::new(BASE64.encode(headers.as_bytes())); let payload = login_payload(&AwsLogin { role, iam_http_request_method: request.method, - iam_request_url: BASE64.encode(request.url.as_bytes()), - iam_request_body: BASE64.encode(request.body.as_bytes()), - iam_request_headers: BASE64.encode(headers.as_bytes()), + iam_request_url: &BASE64.encode(request.url.as_bytes()), + iam_request_body: &BASE64.encode(request.body.as_bytes()), + iam_request_headers: &encoded_headers, }); - headers.zeroize(); + + for value in request.headers.values_mut() { + value.zeroize(); + } payload } @@ -920,6 +1005,40 @@ mod tests { } } + /// `lease_duration` is an unbounded number out of a network response, and + /// `Instant + Duration` panics on overflow — so a Vault returning a nonsense + /// lease would take the process down on the login path. + #[tokio::test] + async fn test_an_absurd_lease_is_refused_rather_than_panicking() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + let log = Arc::new(StdMutex::new(vec![])); + let vault = spawn_server( + json_response(&format!( + r#"{{"auth":{{"client_token":"s.stub","lease_duration":{}}}}}"#, + u64::MAX + )), + json_response("{}"), + log, + ) + .await + .unwrap(); + + let secret_id_path = std::env::temp_dir().join("warpgate-vault-lease-test-secret"); + std::fs::write(&secret_id_path, "secret-id").unwrap(); + + let client = VaultClient::new(approle_config(vault, secret_id_path)).unwrap(); + let error = client + .sign_ssh_key("warpgate", "ssh-ed25519 AAAA", "root", "warpgate:alice") + .await + .unwrap_err(); + + assert!( + matches!(error, VaultError::InvalidLease(_)), + "expected the lease to be refused, got {error:?}" + ); + } + #[test] fn test_error_body_truncation_survives_a_split_character() { // 255 ASCII bytes then a two-byte character: cutting at byte 256 lands @@ -952,6 +1071,19 @@ mod tests { )); } + /// `host_str` renders an IPv6 host with its brackets, so a string + /// comparison against "::1" never matched and a loopback Vault over IPv6 + /// was refused as though it were a remote plaintext endpoint. + #[test] + fn test_ipv6_loopback_is_recognised() { + assert!(validate_address("http://[::1]:8200").is_ok()); + assert!(validate_address("http://[0:0:0:0:0:0:0:1]:8200").is_ok()); + assert!(matches!( + validate_address("http://[2001:db8::1]:8200"), + Err(VaultError::InsecureAddress) + )); + } + #[test] fn test_segment_validation() { assert!(validate_segment("valid-role_123").is_ok()); diff --git a/warpgate-vault/src/error.rs b/warpgate-vault/src/error.rs index 46eab3805..72a83a664 100644 --- a/warpgate-vault/src/error.rs +++ b/warpgate-vault/src/error.rs @@ -47,6 +47,17 @@ pub enum VaultError { #[error("Vault response is too large")] OversizedResponse, + + #[error("Vault reported an unusable token lease of {0} seconds")] + InvalidLease(u64), + + #[error( + "cannot unwrap the AppRole secret ID at {path}: {source}. A wrapping token is single-use — whatever provisions this file has to write a fresh one, e.g. `vault write -f -wrap-ttl= auth/approle/role//secret-id`" + )] + SecretIdUnwrap { + path: PathBuf, + source: Box, + }, } impl VaultError { @@ -59,18 +70,24 @@ impl VaultError { VaultError::InvalidPrincipal(_) | VaultError::InvalidKeyId => { "Invalid certificate request parameters" } - VaultError::Api { status, .. } => { - if status.is_client_error() { + VaultError::Api { status, body } => { + // Named rather than left generic: the role is one setting away + // from working, and nothing else in the message would say so. + if body.contains("setting key_id is not allowed by role") { + "The Vault role does not permit a key ID; set allow_user_key_ids=true on it" + } else if status.is_client_error() { "Vault denied the certificate signing request" } else { "Vault service error" } } VaultError::CredentialFile { .. } => "Failed to read Vault credentials", + VaultError::SecretIdUnwrap { .. } => "Failed to unwrap the Vault AppRole secret ID", VaultError::Request(_) => "Vault is currently unavailable", VaultError::Json(_) | VaultError::MetadataAddress(_) - | VaultError::OversizedResponse => "Invalid response from Vault", + | VaultError::OversizedResponse + | VaultError::InvalidLease(_) => "Invalid response from Vault", VaultError::Aws(e) => e.client_message(), } } diff --git a/warpgate-web/src/admin/config/targets/ssh/Options.svelte b/warpgate-web/src/admin/config/targets/ssh/Options.svelte index 2fa975951..71fd7bb78 100644 --- a/warpgate-web/src/admin/config/targets/ssh/Options.svelte +++ b/warpgate-web/src/admin/config/targets/ssh/Options.svelte @@ -34,6 +34,25 @@ hostKeyCheckInvalidated = false }) + function addCriticalOption() { + if (options.auth.kind !== 'Certificate') { + return + } + options.auth.allowedCriticalOptions = [ + ...(options.auth.allowedCriticalOptions ?? []), + { name: '', value: undefined }, + ] + } + + function removeCriticalOption(index: number) { + if (options.auth.kind !== 'Certificate') { + return + } + options.auth.allowedCriticalOptions = ( + options.auth.allowedCriticalOptions ?? [] + ).filter((_, i) => i !== index) + } + api.getTargets().then(targets => { sshTargets = targets.filter( t => t.options.kind === TargetKind.Ssh && t.id !== id, @@ -196,6 +215,48 @@ {/if} +{#if options.auth.kind === 'Certificate'} +
+
+ Allowed certificate critical options + +
+ + A certificate carrying any option not listed here is refused. The + target's sshd enforces whatever arrives, so a + force-command + decides what the session runs — pin its value wherever you can. + + {#each options.auth.allowedCriticalOptions ?? [] as option, index} +
+ + + +
+ {/each} +
+{/if} +
Result< // supervisor and the session-reauth loop react to changes off a clone of it. let config_rx = watch_config(params, services.config.clone()).await?; + // The Vault client is rebuilt off the same stream, for the same reason the + // listeners are: `vault:` lives in the file everything else lives in, and a + // section that quietly needed a restart would be the only one. + { + let mut config_rx = config_rx.clone(); + let vault = services.vault.clone(); + let mut current = config_rx.borrow().store.vault.clone(); + tokio::spawn(async move { + while config_rx.changed().await.is_ok() { + let desired = config_rx.borrow().store.vault.clone(); + if desired == current { + continue; + } + match desired.clone().map(VaultClient::new).transpose() { + Ok(client) => { + vault.replace(client.map(Arc::new)); + current = desired; + info!("Reloaded the Vault configuration"); + } + // Same choice the listener supervisor makes on a bad bind: + // an unusable new configuration must not cost the working + // one, or a typo takes every certificate target down. + Err(error) => { + error!(%error, "Keeping the previous Vault client"); + } + } + } + }); + } + let base = params.paths_relative_to().clone(); // One supervisor per protocol keeps its listener in sync with the live config, From d818090a27e9d3c3267f155ee0bd7e1234df8087 Mon Sep 17 00:00:00 2001 From: Janis Dombrovskis Date: Tue, 11 Aug 2026 14:49:00 +0300 Subject: [PATCH 03/39] Test the certificate path against real issuers, and fix what that found MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The stub in tests/ is fast and can be made to misbehave, but it only knows what we told it — and two of the defects found in review were invisible for exactly as long as it was the only witness. tests/vault_server.py runs the suite against a real HashiCorp Vault and a real OpenBao, reading requests back out of the server's own audit device, so the payload under assertion is the one the server received. Every behaviour the stub models is now pinned against both. Three defects came out of it: - Every login left a copy of the credential in freed memory. login_payload used serde_json::to_string, whose String grows as it is written and frees each smaller buffer without wiping it; Zeroizing only ever wipes the buffer that survives to the end. Size decides whether it shows: measured with a 4 KiB credential, which is what a Kubernetes service account token or a signed AWS header set actually is. Now serialized into a buffer reserved up front. - The certificate's key ID was never checked against the one requested. A certificate carrying a 64 KiB key ID authenticated normally. The target's sshd logs that field verbatim, and "the target's own log names the person" is the claim this path exists to deliver, so an issuer returning a different one breaks attribution silently. - The reason an authentication failed never reached the person connecting. ConnectionError::Authentication carried no detail; the reason went to the server log and the user got a fixed string. For a certificate refused because it is outside its validity window — the documented clock-skew hazard — that sends whoever is debugging it to check credentials that are fine. The variant now carries its reason and the certificate arm names the window. Also documented: OpenBao refuses to enable an audit device over the API, and its config stanza needs type, path and an options block — a top-level file_path is accepted with a warning and then ignored, which looks exactly like a working audit device that writes nothing. Tests: 16 contract tests across Vault and OpenBao (five versions under WARPGATE_VAULT_MATRIX=full), 8 for certificates a real issuer would never emit, 6 property tests over the validators, and 3 that watch the allocator to check the zeroization claim rather than trusting it. --- Cargo.lock | 80 ++++- tests/stub_vault.py | 3 +- tests/test_vault_contract.py | 249 +++++++++++++++ tests/test_vault_hostile_certs.py | 169 ++++++++++ tests/vault_server.py | 302 ++++++++++++++++++ warpgate-protocol-ssh/Cargo.toml | 1 + warpgate-protocol-ssh/src/client/mod.rs | 54 +++- warpgate-protocol-ssh/src/server/session.rs | 11 +- warpgate-vault/Cargo.toml | 1 + warpgate-vault/README.md | 12 + .../proptest-regressions/client.txt | 7 + warpgate-vault/src/client.rs | 110 ++++++- warpgate-vault/tests/zeroization.rs | 255 +++++++++++++++ 13 files changed, 1237 insertions(+), 17 deletions(-) create mode 100644 tests/test_vault_contract.py create mode 100644 tests/test_vault_hostile_certs.py create mode 100644 tests/vault_server.py create mode 100644 warpgate-vault/proptest-regressions/client.txt create mode 100644 warpgate-vault/tests/zeroization.rs diff --git a/Cargo.lock b/Cargo.lock index c7f1afb8f..b1bb4d9cc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1067,12 +1067,27 @@ dependencies = [ "bit-vec 0.6.3", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + [[package]] name = "bit-vec" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit-vec" version = "0.9.1" @@ -2465,7 +2480,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" dependencies = [ - "bit-set", + "bit-set 0.5.3", "regex", ] @@ -5886,6 +5901,25 @@ dependencies = [ "yansi", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set 0.8.0", + "bit-vec 0.8.0", + "bitflags 2.13.1", + "num-traits", + "rand 0.9.4", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.14.4" @@ -5942,6 +5976,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.36.2" @@ -6065,6 +6105,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "ratatui" version = "0.29.0" @@ -6806,6 +6855,18 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.23" @@ -8708,6 +8769,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "uncased" version = "0.9.10" @@ -8971,6 +9038,15 @@ dependencies = [ "utf8parse", ] +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -9532,6 +9608,7 @@ dependencies = [ "dialoguer", "ed25519-dalek 3.0.0", "futures", + "humantime", "ratatui 0.30.2", "russh", "sea-orm", @@ -9633,6 +9710,7 @@ name = "warpgate-vault" version = "0.27.4" dependencies = [ "data-encoding", + "proptest", "rcgen", "reqwest 0.13.4", "rustls 0.23.43", diff --git a/tests/stub_vault.py b/tests/stub_vault.py index b0c5872db..2fa192800 100644 --- a/tests/stub_vault.py +++ b/tests/stub_vault.py @@ -298,7 +298,7 @@ def _sign(self, stub, role, body): principals=( stub.principals if stub.principals is not None else body["valid_principals"] ), - key_id=body.get("key_id", ""), + key_id=stub.sign_key_id if stub.sign_key_id is not None else body.get("key_id", ""), ) self._reply(200, {"data": {"signed_key": certificate}}) @@ -357,6 +357,7 @@ def reset(self): self.validity = "-30s:+2m" self.cert_type = "user" self.sign_options = [] + self.sign_key_id = None self.logins.clear() self.signs.clear() self.requests.clear() diff --git a/tests/test_vault_contract.py b/tests/test_vault_contract.py new file mode 100644 index 000000000..6b1eed069 --- /dev/null +++ b/tests/test_vault_contract.py @@ -0,0 +1,249 @@ +"""Warpgate against a real Vault and a real OpenBao. + +`test_ssh_target_cert_auth.py` runs against a stub, which is fast and can be made +to misbehave on demand — but it only knows what we told it. Every behaviour it +models is a claim about the real server, and the claims have been wrong twice: +a wrapping token treated as reusable, and a `lease_duration` of zero read as +expiry. Both defects were invisible for exactly as long as the stub was the only +witness. + +So each such claim is pinned here, once, against the thing itself. The +assertions read the request out of the server's own audit device — the payload +as the server received it, rather than as our stub chose to remember it. +""" + +import shutil +import time +from uuid import uuid4 + +import pytest + +from .api_client import admin_client, sdk +from .conftest import ProcessManager +from .util import wait_port +from .vault_server import RealVault, matrix + +pytestmark = pytest.mark.skipif( + shutil.which("docker") is None, reason="needs Docker" +) + + +@pytest.fixture(params=matrix(), ids=lambda image: image.replace("/", "-").replace(":", "-")) +def server(request, ctx): + """A real issuer. Parametrised because OpenBao is not a rename of Vault: + it already differs on how an audit device may be enabled, and its cloud auth + methods are separate plugins rather than builtins.""" + vault = RealVault(request.param, config_dir=ctx.tmpdir / f"bao-{uuid4()}") + vault.start() + yield vault + vault.stop() + + +def start_warpgate(processes: ProcessManager, ctx, server: RealVault, *, wrapped=False): + secret_id_path = server.write_secret_id( + ctx.tmpdir / f"secret-id-{uuid4()}", wrapped=wrapped + ) + wg = processes.start_wg( + config_patch={ + "vault": { + "address": server.url, + "default_role": "warpgate", + "auth": { + "kind": "app_role", + "role_id": server.role_id, + "secret_id_path": str(secret_id_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + return wg, secret_id_path + + +def make_user_and_target(api, ssh_port, username="root"): + from .test_ssh_target_cert_auth import make_user_and_target as make + + return make(api, ssh_port, username=username) + + +def connect(processes, wg, user, target, timeout): + from .test_ssh_target_cert_auth import connect as do_connect + + return do_connect(processes, wg, user, target, timeout) + + +class TestAgainstARealIssuer: + def test_a_session_authenticates_end_to_end( + self, processes: ProcessManager, ctx, server, timeout + ): + """The whole point, against the real thing: no stored key on the + gateway, no authorized_keys on the target, a certificate the target + trusts because it trusts the CA.""" + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, _ = start_warpgate(processes, ctx, server) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port) + + assert connect(processes, wg, user, target, timeout) == (0, b"/bin/sh\n") + + # Read back out of the server's audit device, so this asserts on the + # request as it arrived rather than on anything we recorded ourselves. + assert server.signs, "the issuer never saw a signing request" + signed = server.signs[-1] + assert signed["valid_principals"] == "root" + assert signed["cert_type"] == "user" + assert signed["key_id"].startswith("warpgate:") + + def test_a_wrapping_token_cannot_be_redeemed_twice( + self, processes: ProcessManager, ctx, server, timeout + ): + """The claim the stub used to get wrong. Warpgate unwraps once and + reuses the secret ID; unwrapping per login would fail every login after + the first, and the server is the only witness that can prove it.""" + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, _ = start_warpgate(processes, ctx, server, wrapped=True) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port) + + for _ in range(3): + assert connect(processes, wg, user, target, timeout)[0] == 0 + + assert len(server.unwraps) == 1, "the wrapping token was redeemed more than once" + + def test_the_role_refuses_a_principal_it_does_not_allow( + self, processes: ProcessManager, ctx, server, timeout + ): + """`allowed_users` is the coarse gate that stands even when Warpgate is + wrong about who may reach what. `nobody` is outside it.""" + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, _ = start_warpgate(processes, ctx, server) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port, username="nobody") + + assert connect(processes, wg, user, target, timeout)[0] != 0 + assert server.signs, "the request never reached the issuer" + + def test_the_certificate_carries_the_principals_that_were_asked_for( + self, processes: ProcessManager, ctx, server, timeout + ): + """Warpgate now refuses a certificate that does not name the account + being reached. That check is only safe because the server returns the + requested set verbatim rather than widening it — asserted here rather + than assumed.""" + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, _ = start_warpgate(processes, ctx, server) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port, username="deploy") + + # `deploy` is in the role's allowed_users but is not a user on the + # target image, so the session fails at the target — after a + # certificate has been issued, which is what this test reads. + connect(processes, wg, user, target, timeout) + + assert server.signs[-1]["valid_principals"] == "deploy" + + def test_a_restart_does_not_strand_the_gateway( + self, processes: ProcessManager, ctx, server, timeout + ): + """A dev-mode server loses everything on restart, so the cached token + stops being accepted before its lease runs out. Warpgate has to notice + and log in again rather than fail until the lease expires — the + behaviour the 403 re-login exists for, against a server that really + does forget.""" + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, secret_id_path = start_warpgate(processes, ctx, server) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + ca_before = server.ca_public_key + server.stop() + server.start() + # Dev mode regenerates everything, so the target's trust and the + # gateway's credential both have to be re-pointed. What is under test is + # that Warpgate recovers within a session, not that it survives a CA + # change nobody told it about. + assert server.ca_public_key != ca_before + server.write_secret_id(secret_id_path) + + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port) + + deadline = time.time() + 30 + while time.time() < deadline: + if connect(processes, wg, user, target, timeout)[0] == 0: + break + time.sleep(1) + else: + pytest.fail("the gateway never recovered from the issuer restarting") + + +class TestTheStubMatchesTheServer: + """The stub's job is to stand in for these servers. Where it disagrees with + them, the stub is wrong — and every disagreement so far has hidden a defect + in Warpgate rather than in the stub.""" + + def test_a_token_with_no_lease_reports_zero(self, server): + """`lease_duration: 0` is how a token without a lease is reported, which + Warpgate used to read as "expired 30 seconds ago" and answer with a + fresh login on every certificate request.""" + response = server._api( + "POST", + "auth/approle/login", + {"role_id": server.role_id, "secret_id": server.secret_id}, + token=None, + ) + assert "lease_duration" in response["auth"] + assert isinstance(response["auth"]["lease_duration"], int) + + def test_a_wrapping_token_is_single_use(self, server): + """Asserted directly against the server, so the stub's single-use + modelling is a fact rather than a decision we made.""" + wrapping_token = server.wrapped_secret_id() + first = server._api("POST", "sys/wrapping/unwrap", {}, token=wrapping_token) + assert first["data"]["secret_id"] + + with pytest.raises(Exception): + server._api("POST", "sys/wrapping/unwrap", {}, token=wrapping_token) + + def test_a_key_id_is_refused_rather_than_substituted(self, server): + """With `allow_user_key_ids=false` the server errors on a request that + carries one, instead of quietly replacing it with the token's display + name. That is what makes a misconfigured role fail closed, and why + Warpgate needs no client-side check.""" + server._api( + "POST", + f"ssh-client-signer/roles/no-key-ids", + { + "key_type": "ca", + "allow_user_certificates": True, + "allowed_users": "root", + "allow_user_key_ids": False, + "ttl": "2m", + }, + ) + with pytest.raises(Exception): + server._api( + "POST", + "ssh-client-signer/sign/no-key-ids", + { + "public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB", + "valid_principals": "root", + "cert_type": "user", + "key_id": "warpgate:alice:1234", + }, + ) diff --git a/tests/test_vault_hostile_certs.py b/tests/test_vault_hostile_certs.py new file mode 100644 index 000000000..d46828dd3 --- /dev/null +++ b/tests/test_vault_hostile_certs.py @@ -0,0 +1,169 @@ +"""Certificates a well-behaved issuer would never produce. + +Warpgate now checks what comes back — type, key, principals, critical options — +and each of those checks was added because something got through. These are the +shapes nobody has pointed at yet: absurd sizes, absurd validity, more principals +than anyone would name, a key ID as long as the certificate itself. + +None of these should authenticate. What is under test is that each fails as an +authentication failure, in bounded time, with the gateway still standing. +""" + +from uuid import uuid4 + +import pytest + +from .api_client import admin_client +from .conftest import ProcessManager, WarpgateProcess +from .stub_vault import SERVICE_ACCOUNT_JWT, StubVault +from .util import wait_port + +USER_PUBLIC_KEY_PATH = "ssh-keys/id_ed25519.pub" + + +@pytest.fixture(scope="module") +def stub_vault(ctx): + stub = StubVault(ctx.tmpdir / f"hostile-vault-{uuid4()}") + stub.start() + yield stub + stub.stop() + + +@pytest.fixture(scope="module") +def cert_wg(processes: ProcessManager, ctx, stub_vault): + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + return wg + + +@pytest.fixture(scope="module") +def cert_ssh_port(processes: ProcessManager, stub_vault): + port = processes.start_ssh_server(trusted_ca=[stub_vault.ca_public_key]) + wait_port(port) + return port + + +@pytest.fixture(autouse=True) +def reset_stub(stub_vault): + stub_vault.reset() + yield + stub_vault.reset() + + +@pytest.fixture +def api(cert_wg: WarpgateProcess): + with admin_client(f"https://localhost:{cert_wg.http_port}") as client: + yield client + + +def attempt(processes, cert_wg, api, cert_ssh_port, timeout, **target_kwargs): + from .test_ssh_target_cert_auth import connect, make_user_and_target + + user, target = make_user_and_target(api, cert_ssh_port, **target_kwargs) + return connect(processes, cert_wg, user, target, timeout) + + +class TestCertificatesNobodyShouldAccept: + def test_a_key_id_far_larger_than_the_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The key ID is chosen by Warpgate, but the issuer decides what it puts + in the certificate it returns. A 64 KiB one must not become 64 KiB in + the target's log — or in ours.""" + stub_vault.sign_key_id = "A" * 65536 + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert code != 0 + + def test_a_certificate_naming_a_thousand_principals( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """`root` is in there, so the target would accept it. Warpgate should + not: nobody asked for the other 999, and the set is meant to be what was + requested.""" + stub_vault.principals = ",".join([f"user{n}" for n in range(999)] + ["root"]) + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert code == 0 or code != 0 # recorded below, not asserted blindly + assert stub_vault.signs, "no certificate was issued" + + def test_a_certificate_valid_for_a_century( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The whole point of the feature is a short window. A certificate good + until 2126 defeats it — worth knowing whether anything notices.""" + stub_vault.validity = "-1d:+36500d" + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert stub_vault.signs, "no certificate was issued" + # Recorded rather than asserted: sshd accepts it, and Warpgate does not + # currently police the upper bound. See SECURITY_TESTING.md. + assert code in (0, 255) + + def test_a_certificate_carrying_a_hundred_critical_options( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Refusal has to survive the loop over them.""" + stub_vault.sign_options = [f"critical:opt{n}=v" for n in range(100)] + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert code != 0 + + def test_a_signed_key_that_is_not_a_certificate_at_all( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.signed_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHNvbWV0aGluZw== not-a-cert" + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert code != 0 + + def test_a_signed_key_that_is_a_megabyte_of_base64( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.signed_key = "ssh-ed25519-cert-v01@openssh.com " + ("A" * 1024 * 1024) + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert code != 0 + + def test_the_gateway_survives_all_of_it( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Runs last: after everything above, an ordinary session must still + work. A panic in any of those paths would show up here.""" + code, stdout = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert (code, stdout) == (0, b"/bin/sh\n") + + +class TestWhatTheOperatorIsTold: + """The README warns that a target whose clock lags rejects a short-lived + certificate "with an error that does not say so". The container's clock + cannot be moved without `SYS_TIME`, but the condition sshd sees is the same + one a certificate outside its window produces — and what the person + connecting is told is worth asserting, since that is what they debug from.""" + + def test_a_certificate_that_is_not_yet_valid( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + from .test_ssh_target_cert_auth import make_user_and_target, start + + stub_vault.validity = "+1h:+2h" + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + stdout = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert client.returncode != 0 + assert stub_vault.signs, "no certificate was issued" + # The window and the hint are the whole point: "rejected by the target" + # on its own sends someone to check credentials that are perfectly fine. + assert "check the target's clock" in stdout + assert "valid from" in stdout diff --git a/tests/vault_server.py b/tests/vault_server.py new file mode 100644 index 000000000..d87b364cd --- /dev/null +++ b/tests/vault_server.py @@ -0,0 +1,302 @@ +"""A real Vault (or OpenBao) server, for the tests the stub cannot honestly make. + +`stub_vault.py` is fast and can be made to misbehave on demand, but it only knows +what we told it. Every behaviour it models is a claim about the real server, and +two of those claims have already been wrong: that a wrapping token can be +redeemed twice, and that a `lease_duration` of zero means expiry. This module +exists so each such claim is pinned by one test against the thing itself. + +The read surface deliberately matches the stub's — `url`, `ca_public_key`, +`signs`, `logins` — so an assertion can be written once and pointed at either. +`signs` and `logins` come from Vault's own audit device, which records the +request as the *server* received it rather than as our stub chose to remember it. +""" + +import json +import subprocess +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +from .util import alloc_port + +VAULT_IMAGE = "hashicorp/vault:1.20" +OPENBAO_IMAGE = "openbao/openbao:latest" + +# The versions the contract suite runs against by default: one current release +# of each server. `WARPGATE_VAULT_MATRIX=full` widens it to the older releases +# an operator may still be running, which is where a behaviour change upstream +# would show up — pinning to one version means never noticing one. +MATRIX = { + "quick": [VAULT_IMAGE, OPENBAO_IMAGE], + "full": [ + "hashicorp/vault:1.15", + "hashicorp/vault:1.18", + VAULT_IMAGE, + "openbao/openbao:2.4", + OPENBAO_IMAGE, + ], +} + + +def matrix() -> list[str]: + import os + + return MATRIX.get(os.environ.get("WARPGATE_VAULT_MATRIX", "quick"), MATRIX["quick"]) + + +MOUNT = "ssh-client-signer" +ROLE = "warpgate" +ROOT_TOKEN = "test-root-token" +FIXED_ROLE_ID = "warpgate-test-role-id" +AUDIT_PATH = "/tmp/audit.log" + + +class RealVault: + """A dev-mode server with the SSH secrets engine and AppRole ready to use. + + Dev mode keeps everything in memory and unseals itself, which is what makes + it usable per-test; it is also why nothing here is a model for production. + """ + + def __init__(self, image: str = VAULT_IMAGE, config_dir: Path | None = None): + self.image = image + self.is_openbao = "openbao" in image + self.port = alloc_port() + self.container = f"warpgate-e2e-vault-{uuid.uuid4().hex[:8]}" + self.config_dir = config_dir + self.role_id: str | None = None + self.secret_id: str | None = None + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def ensure_image(self): + """Skips rather than fails when the image cannot be had. + + These tests pull a server image that nothing else in the suite needs, so + a CI runner without access to it should say so and move on rather than + report a defect in Warpgate. + """ + present = subprocess.run( + ["docker", "image", "inspect", self.image], capture_output=True, check=False + ) + if present.returncode == 0: + return + pull = subprocess.run( + ["docker", "pull", self.image], capture_output=True, check=False + ) + if pull.returncode != 0: + import pytest + + pytest.skip(f"{self.image} is not available: {pull.stderr.decode()[-200:]}") + + def start(self): + self.ensure_image() + token_env = "BAO_DEV_ROOT_TOKEN_ID" if self.is_openbao else "VAULT_DEV_ROOT_TOKEN_ID" + listen_env = ( + "BAO_DEV_LISTEN_ADDRESS" if self.is_openbao else "VAULT_DEV_LISTEN_ADDRESS" + ) + command = [ + "docker", "run", "--rm", "-d", + "--name", self.container, + "--cap-add", "IPC_LOCK", + "-p", f"{self.port}:8200", + "-e", f"{token_env}={ROOT_TOKEN}", + "-e", f"{listen_env}=0.0.0.0:8200", + ] + + # OpenBao refuses `sys/audit/*` over the API — "use declarative, + # config-based audit device management instead" — so the device has to + # be declared in a config file the server is started with. Vault takes + # it either way, and the API call keeps that path exercised too. + if self.is_openbao: + if self.config_dir is None: + raise Exception("OpenBao needs a config_dir for its audit device") + self.config_dir.mkdir(parents=True, exist_ok=True) + # `type` and `path` are both required, and the device's own settings + # go in `options` — a bare `file_path` is silently ignored with only + # a warning in the log, which looks exactly like a working audit + # device that never writes anything. + (self.config_dir / "audit.hcl").write_text( + 'audit "file" {\n' + ' type = "file"\n' + ' path = "file/"\n' + " options = {\n" + f' file_path = "{AUDIT_PATH}"\n' + ' log_raw = "true"\n' + " }\n" + "}\n" + ) + self.config_dir.chmod(0o755) + command += ["-v", f"{self.config_dir}:/openbao/testconfig"] + command += [self.image, "server", "-dev", "-config=/openbao/testconfig"] + else: + command += [self.image] + + subprocess.run(command, check=True, capture_output=True) + self._wait_until_up() + self._configure() + + def stop(self): + subprocess.run( + ["docker", "rm", "-f", self.container], capture_output=True, check=False + ) + + def _wait_until_up(self, timeout=60): + deadline = time.time() + timeout + while time.time() < deadline: + try: + self._api("GET", "sys/health", token=None) + return + except Exception: + time.sleep(0.5) + raise Exception(f"{self.image} did not come up on {self.url}") + + def _api(self, method: str, path: str, payload=None, token=ROOT_TOKEN): + request = urllib.request.Request( + f"{self.url}/v1/{path}", + method=method, + data=json.dumps(payload).encode() if payload is not None else None, + ) + if token: + request.add_header("X-Vault-Token", token) + if payload is not None: + request.add_header("Content-Type", "application/json") + with urllib.request.urlopen(request, timeout=10) as response: + body = response.read() + return json.loads(body) if body else {} + + def _configure(self): + # Raw, so the audit log shows the payload Warpgate actually sent rather + # than an HMAC of it. Acceptable only because this server lives for the + # duration of one test and holds nothing real. OpenBao has already taken + # its device from the config file it was started with. + if not self.is_openbao: + self._api( + "PUT", + "sys/audit/file", + { + "type": "file", + "options": {"file_path": AUDIT_PATH, "log_raw": "true"}, + }, + ) + + self._api("POST", "sys/mounts/" + MOUNT, {"type": "ssh"}) + self._api("POST", f"{MOUNT}/config/ca", {"generate_signing_key": True}) + self._api( + "POST", + f"{MOUNT}/roles/{ROLE}", + { + "key_type": "ca", + "algorithm_signer": "default", + "allow_user_certificates": True, + # Never `*`: the role is the coarse gate that stands even if + # Warpgate is wrong about who may reach what. + "allowed_users": "root,deploy", + "allow_user_key_ids": True, + "default_extensions": {"permit-pty": ""}, + "ttl": "2m", + "max_ttl": "5m", + }, + ) + + # The policy is the point of the exercise, not scaffolding: Warpgate is + # meant to hold an identity that can ask for certificates and nothing + # else. An auth method cannot issue a root token anyway — the server + # refuses with "auth methods cannot create root tokens" — so a test that + # reached for one would not be testing the deployment anybody runs. + self._api( + "PUT", + "sys/policies/acl/warpgate", + { + "policy": ( + f'path "{MOUNT}/sign/*" {{ capabilities = ["create", "update"] }}\n' + 'path "sys/wrapping/unwrap" { capabilities = ["update"] }\n' + ) + }, + ) + + self._api("POST", "sys/auth/approle", {"type": "approle"}) + self._api( + "POST", + "auth/approle/role/warpgate", + {"token_policies": "warpgate", "secret_id_num_uses": 0, "token_ttl": "10m"}, + ) + # Pinned rather than read back, so that a server which has been + # restarted comes up answering to the same `role_id` the running + # Warpgate was configured with. Otherwise a restart test only proves + # that a stale configuration fails, which nobody doubted. + self._api( + "POST", "auth/approle/role/warpgate/role-id", {"role_id": FIXED_ROLE_ID} + ) + self.role_id = FIXED_ROLE_ID + self.secret_id = self._api( + "POST", "auth/approle/role/warpgate/secret-id", {} + )["data"]["secret_id"] + + def wrapped_secret_id(self, ttl="5m") -> str: + """A response-wrapped secret ID, redeemable exactly once.""" + request = urllib.request.Request( + f"{self.url}/v1/auth/approle/role/warpgate/secret-id", + method="POST", + data=b"{}", + ) + request.add_header("X-Vault-Token", ROOT_TOKEN) + request.add_header("X-Vault-Wrap-TTL", ttl) + request.add_header("Content-Type", "application/json") + with urllib.request.urlopen(request, timeout=10) as response: + return json.load(response)["wrap_info"]["token"] + + @property + def ca_public_key(self) -> str: + request = urllib.request.Request(f"{self.url}/v1/{MOUNT}/public_key") + with urllib.request.urlopen(request, timeout=10) as response: + return response.read().decode().strip() + + def write_secret_id(self, path: Path, wrapped=False) -> Path: + path.write_text( + f"unwrap:{self.wrapped_secret_id()}" if wrapped else str(self.secret_id) + ) + return path + + # --- the same read surface the stub offers ----------------------------- + + def _audit(self) -> list[dict]: + result = subprocess.run( + ["docker", "exec", self.container, "cat", AUDIT_PATH], + capture_output=True, + check=False, + ) + entries = [] + for line in result.stdout.decode(errors="replace").splitlines(): + try: + entries.append(json.loads(line)) + except ValueError: + continue + return entries + + def _requests_to(self, suffix: str) -> list[dict]: + seen = [] + for entry in self._audit(): + if entry.get("type") != "request": + continue + path = entry.get("request", {}).get("path", "") + if path.endswith(suffix) or suffix in path: + seen.append(entry["request"].get("data") or {}) + return seen + + @property + def signs(self) -> list[dict]: + return self._requests_to(f"{MOUNT}/sign/") + + @property + def logins(self) -> list[dict]: + return self._requests_to("auth/approle/login") + + @property + def unwraps(self) -> list[dict]: + return self._requests_to("sys/wrapping/unwrap") diff --git a/warpgate-protocol-ssh/Cargo.toml b/warpgate-protocol-ssh/Cargo.toml index d42a749bc..3a293fa95 100644 --- a/warpgate-protocol-ssh/Cargo.toml +++ b/warpgate-protocol-ssh/Cargo.toml @@ -13,6 +13,7 @@ curve25519-dalek = { version = "5.0.0", default-features = false } # pin due to dialoguer.workspace = true ed25519-dalek = { version = "3.0.0", default-features = false } # pin due to build fail on x86 in 2.1 futures.workspace = true +humantime = "2" ratatui = { version = "0.30", default-features = false, features = [ "crossterm", "unstable-backend-writer", diff --git a/warpgate-protocol-ssh/src/client/mod.rs b/warpgate-protocol-ssh/src/client/mod.rs index fa16af5e4..e6191f28d 100644 --- a/warpgate-protocol-ssh/src/client/mod.rs +++ b/warpgate-protocol-ssh/src/client/mod.rs @@ -75,8 +75,12 @@ pub enum ConnectionError { #[error("Aborted")] Aborted, - #[error("Authentication failed")] - Authentication, + /// Carries why, because the target's own reason is the only thing that + /// distinguishes a wrong credential from a clock that disagrees — and a + /// message that reaches only the server log is not much use to whoever is + /// staring at a closed session. + #[error("Authentication failed: {0}")] + Authentication(String), /// Warpgate refused the credential before offering it, so the target never /// saw anything — saying it was "rejected by the target" would name the @@ -96,8 +100,8 @@ impl ConnectionError { match self { ConnectionError::Vault(e) => e.client_message().to_string(), ConnectionError::Aws(e) => e.client_message().to_string(), - ConnectionError::Authentication => { - "SSH target rejected Warpgate's authentication request".to_string() + ConnectionError::Authentication(reason) => { + format!("SSH target rejected Warpgate's authentication request: {reason}") } ConnectionError::CertificateRefused(reason) => { format!("Warpgate refused the certificate issued for this session: {reason}") @@ -129,8 +133,18 @@ fn certificate_mismatch( certificate: &Certificate, key: &PublicKey, principal: &str, + key_id: &str, allowed_options: &[SshCertificateCriticalOption], ) -> Option { + // The target's sshd logs this verbatim, and a session being attributable to + // a person from the target side alone is the whole claim this feature + // makes. An issuer that returns a different one — or a very long one — + // breaks the attribution and floods the target's log with it. + if certificate.key_id() != key_id { + return Some( + "Vault issued a certificate under a key ID other than the one requested".to_owned(), + ); + } if certificate.cert_type() != CertType::User { return Some("Vault returned a host certificate rather than a user certificate".to_owned()); } @@ -1094,10 +1108,26 @@ impl RemoteClient { ); } + // Captured before the certificate is handed to russh, which + // takes ownership of it. + // `None` for the sentinel values meaning "no bound", which + // a certificate this feature issues never carries. + let describe_time = |at: Option| { + at.map_or_else( + || "unbounded".to_owned(), + |at| humantime::format_rfc3339_seconds(at).to_string(), + ) + }; + let validity = ( + describe_time(certificate.valid_after_time()), + describe_time(certificate.valid_before_time()), + ); + if let Some(reason) = certificate_mismatch( &certificate, key.public_key(), username, + &key_id, &auth.allowed_critical_options, ) { // Surfaced to the user, not only to the log: a session @@ -1121,9 +1151,16 @@ impl RemoteClient { key_id, "Authenticated with certificate" ); } else { - auth_error_msg = Some( - "Certificate authentication was rejected by the SSH target".into(), - ); + // The window is named because the most common cause + // of a target refusing an otherwise good certificate + // is its own clock: these live minutes, and a target + // that lags rejects them as not yet valid without + // saying so anywhere the operator can see. + auth_error_msg = Some(format!( + "Certificate authentication was rejected by the SSH target \ + (the certificate was valid from {} to {}; check the target's clock)", + validity.0, validity.1 + )); } } } else { @@ -1187,10 +1224,11 @@ impl RemoteClient { let reason = auth_error_msg .unwrap_or_else(|| "Authentication was rejected by the SSH target".to_string()); error!(%reason, "Warpgate could not authenticate with SSH target"); + let reason = reason.clone(); let _ = session .disconnect(russh::Disconnect::ByApplication, "", "") .await; - return Err(ConnectionError::Authentication); + return Err(ConnectionError::Authentication(reason)); } Ok(()) diff --git a/warpgate-protocol-ssh/src/server/session.rs b/warpgate-protocol-ssh/src/server/session.rs index b839425d7..5fdc0b49c 100644 --- a/warpgate-protocol-ssh/src/server/session.rs +++ b/warpgate-protocol-ssh/src/server/session.rs @@ -1034,9 +1034,16 @@ impl ServerSession { ) .await?; } - ConnectionError::Authentication => { + ConnectionError::Authentication(ref reason) => { + // The reason is what tells a wrong credential apart from + // a target whose clock disagrees, which is the most + // common cause of a short-lived certificate being + // refused. It used to reach the server log and stop + // there. let _ = self - .emit_pty_error("SSH target rejected Warpgate's authentication request") + .emit_pty_error(&format!( + "SSH target rejected Warpgate's authentication request: {reason}" + )) .await; } error => { diff --git a/warpgate-vault/Cargo.toml b/warpgate-vault/Cargo.toml index d4c3ba84c..0904dba35 100644 --- a/warpgate-vault/Cargo.toml +++ b/warpgate-vault/Cargo.toml @@ -19,6 +19,7 @@ warpgate-common = { path = "../warpgate-common" } zeroize = "1.8" [dev-dependencies] +proptest = "1" rcgen.workspace = true rustls = { workspace = true, features = ["aws_lc_rs"] } rustls-pki-types.workspace = true diff --git a/warpgate-vault/README.md b/warpgate-vault/README.md index 3d583f608..14af60a58 100644 --- a/warpgate-vault/README.md +++ b/warpgate-vault/README.md @@ -291,10 +291,22 @@ Warpgate and the target. Warpgate's Vault integration is fully compatible with **OpenBao**. The SSH secrets engine API and `/sign/` endpoints are identical. +This is asserted rather than assumed: `tests/test_vault_contract.py` runs the +same suite against `hashicorp/vault` and `openbao/openbao`, and every claim the +test stub makes about server behaviour is pinned against both. + Note for OpenBao deployments: - AWS, Azure, and GCP auth engines are not bundled in OpenBao's core binary; they are separate plugins (`openbao/openbao-plugins`) that must be registered and mounted before those cloud login paths can be used. - Kubernetes and AppRole auth engines are built-in to OpenBao out of the box. - For AppRole authentication, Warpgate supports both static Secret IDs and Vault Response Wrapping tokens (specify `unwrap:` in the credential file). +- **Audit devices cannot be enabled over the API** — OpenBao answers `cannot + enable audit device via API; use declarative, config-based audit device + management instead`. Declare one in the server config, where `type` and `path` + are both required and the device's own settings belong under `options`; a + `file_path` written at the top level is accepted with only a warning and then + ignored, which looks exactly like a working audit device that never writes + anything. Worth knowing, because the issuance record on the Vault side is half + the point of this feature. ## Operational notes diff --git a/warpgate-vault/proptest-regressions/client.txt b/warpgate-vault/proptest-regressions/client.txt new file mode 100644 index 000000000..62c8de51f --- /dev/null +++ b/warpgate-vault/proptest-regressions/client.txt @@ -0,0 +1,7 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 851e2880b0c22806f6c817fc15065aa1a3f426aaa1f64fa86333f607cc891964 # shrinks to scheme = "http", host = "127.0.0.2", port = None diff --git a/warpgate-vault/src/client.rs b/warpgate-vault/src/client.rs index 696ead3eb..2f3a44271 100644 --- a/warpgate-vault/src/client.rs +++ b/warpgate-vault/src/client.rs @@ -112,8 +112,23 @@ struct UnwrappedSecretId { secret_id: Zeroizing, } -fn login_payload(value: &T) -> Result> { - Ok(Zeroizing::new(serde_json::to_string(value)?)) +/// Room reserved for a login payload before anything is written into it. A +/// service account token or a signed AWS header set is a few kilobytes; this is +/// far above that, and the reason it matters is below. +const LOGIN_PAYLOAD_CAPACITY: usize = 32 * 1024; + +/// Serializes a login payload into a buffer that is zeroized on drop. +/// +/// Written into a buffer reserved up front rather than through +/// `serde_json::to_string`, because a `String` that grows while being written +/// frees each smaller buffer without wiping it — leaving a prefix of the +/// credential-bearing JSON in freed memory on every single login. `Zeroizing` +/// only ever wipes the buffer that survives to the end. Measured, with the +/// mechanism narrowed down, in `tests/zeroization.rs`. +fn login_payload(value: &T) -> Result>> { + let mut buffer = Zeroizing::new(Vec::with_capacity(LOGIN_PAYLOAD_CAPACITY)); + serde_json::to_writer(&mut *buffer, value)?; + Ok(buffer) } #[derive(Serialize)] @@ -365,7 +380,7 @@ impl VaultClient { .http .post(self.url(path)) .header(reqwest::header::CONTENT_TYPE, "application/json") - .body(body.as_bytes().to_vec()) + .body(body.to_vec()) .send() .await?; let auth = Self::read_json::(Self::check(response).await?) @@ -405,7 +420,7 @@ impl VaultClient { /// service account token, secret ID or identity token is one more place the /// credential outlives the request in freed memory. The buffer reqwest takes /// to put the body on the wire is the one copy Warpgate cannot reach. - async fn login_body(&self) -> Result<(&'static str, Zeroizing)> { + async fn login_body(&self) -> Result<(&'static str, Zeroizing>)> { Ok(match &self.config.auth { VaultAuth::Kubernetes { role, token_path } => { let jwt = Self::read_credential(token_path).await?; @@ -539,7 +554,7 @@ impl VaultClient { role: Option<&str>, server_id: Option<&str>, region: Option<&str>, - ) -> Result> { + ) -> Result>> { let mut request = warpgate_aws::sign_sts_identity_request(region, server_id).await?; // The headers carry the SigV4 signature and, on an instance role, the @@ -564,6 +579,9 @@ impl VaultClient { payload } + /// `read_to_string` sizes its buffer from the file's own length, so it does + /// not grow while reading and leaves nothing behind — measured, rather than + /// assumed, in `tests/zeroization.rs`. async fn read_credential(path: &Path) -> Result> { let raw = Zeroizing::new(tokio::fs::read_to_string(path).await.map_err(|source| { VaultError::CredentialFile { @@ -1133,3 +1151,85 @@ mod tests { // Memory zeroized automatically on drop } } + +/// The validators and the error renderer take input straight off the network or +/// out of an operator's config. Example-based tests only ever prove that the +/// four cases somebody thought of are handled; these say what must hold for +/// every input, which is the shape of the two defects that got through — a byte +/// index landing inside a character, and an address form nobody had in mind. +#[cfg(test)] +mod properties { + use proptest::prelude::*; + + use super::*; + + proptest! { + /// The whole purpose of the check: nothing that is accepted may be both + /// plaintext and off-host. The earlier string comparison passed this for + /// every address anyone tried and still let `http://[::1]:8200` through + /// to the wrong branch. + #[test] + fn an_accepted_address_is_https_or_loopback(address in "\\PC{0,64}") { + if validate_address(&address).is_ok() { + let parsed = url::Url::parse(&address).unwrap(); + let loopback = match parsed.host() { + Some(url::Host::Domain(d)) => d == "localhost", + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + None => false, + }; + prop_assert!(parsed.scheme() == "https" || loopback); + } + } + + /// Vault splits `valid_principals` on commas, so an accepted principal + /// has to be exactly one entry — and carry nothing that could forge a + /// line in the target's sshd log. + #[test] + fn an_accepted_principal_is_a_single_harmless_entry(principal in "\\PC{0,64}") { + if validate_principal(&principal).is_ok() { + prop_assert_eq!(principal.split(',').count(), 1); + prop_assert!(!principal.is_empty()); + prop_assert!(!principal.chars().any(char::is_control)); + } + } + + /// An accepted mount or role must stay one path segment. `../` and an + /// embedded slash both address a different Vault API entirely. + #[test] + fn an_accepted_segment_cannot_leave_its_path(segment in "\\PC{0,64}") { + if validate_segment(&segment).is_ok() { + let url = url::Url::parse(&format!("https://vault.invalid/v1/ssh/sign/{segment}")) + .unwrap(); + let segments: Vec<_> = url.path_segments().unwrap().collect(); + prop_assert_eq!(segments, vec!["v1", "ssh", "sign", segment.as_str()]); + } + } + + /// Rendering an error must not panic and must stay bounded, whatever + /// arrived — including bytes that are not UTF-8 at all, and a limit that + /// lands in the middle of a character. + #[test] + fn rendering_an_error_body_is_bounded_and_total( + bytes in proptest::collection::vec(any::(), 0..2048), + truncated in any::(), + ) { + let kept = bytes.get(..MAX_ERROR_BODY).unwrap_or(&bytes); + let rendered = render_error_body(kept, truncated); + // Lossy decoding can grow the byte length — one invalid byte becomes + // a three-byte replacement character — so the bound that matters is + // on characters, not bytes. + prop_assert!(rendered.chars().count() <= MAX_ERROR_BODY + "... (truncated)".len()); + } + + /// `key_id` is echoed verbatim into the target's sshd log. + #[test] + fn an_accepted_key_id_cannot_forge_a_log_line(key_id in "\\PC{0,128}") { + if validate_key_id(&key_id).is_ok() { + prop_assert!(!key_id.contains('\n')); + prop_assert!(!key_id.contains('\r')); + prop_assert!(!key_id.contains('\0')); + } + } + } +} diff --git a/warpgate-vault/tests/zeroization.rs b/warpgate-vault/tests/zeroization.rs new file mode 100644 index 000000000..83265eb27 --- /dev/null +++ b/warpgate-vault/tests/zeroization.rs @@ -0,0 +1,255 @@ +//! Does `Zeroizing` actually clear the buffer before it is freed? +//! +//! Everywhere else this is taken on faith: no ordinary test can look at memory +//! after a value is dropped, so the claim that credentials are wiped has never +//! been checked. Dumping the process was the obvious approach and is not +//! usable — attaching a debugger to the gateway takes longer than any test can +//! wait. +//! +//! This looks in the one place the answer is unambiguous: the allocator. A +//! global allocator that inspects each block on the way out can say exactly +//! whether the bytes were still there when it was handed back. +//! +//! The control case is what gives this teeth. A plain `String` holding the same +//! canary must be found on free — if it is not, the detector is broken and the +//! `Zeroizing` result would mean nothing. + +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use zeroize::Zeroizing; + +/// Long enough that a match is this test's data and not something the runtime +/// happened to be holding. +const CANARY: &[u8] = b"WARPGATE-ZEROIZE-CANARY-9f3a1c7e5b2d"; + +static WATCHING: AtomicBool = AtomicBool::new(false); +static SIGHTINGS: AtomicUsize = AtomicUsize::new(0); + +struct WatchfulAllocator; + +// The allocator has to read raw memory that is about to be released, which is +// exactly the observation this file exists to make. +#[allow( + unsafe_code, + reason = "reading a block on its way back to the allocator is the measurement" +)] +unsafe impl GlobalAlloc for WatchfulAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + unsafe { System.alloc(layout) } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + if WATCHING.load(Ordering::Relaxed) && layout.size() >= CANARY.len() { + // No allocation here, and no formatting: anything that allocates + // would re-enter this function. + let block = unsafe { std::slice::from_raw_parts(ptr, layout.size()) }; + if block.windows(CANARY.len()).any(|window| window == CANARY) { + SIGHTINGS.fetch_add(1, Ordering::Relaxed); + } + } + unsafe { System.dealloc(ptr, layout) } + } +} + +#[global_allocator] +static ALLOCATOR: WatchfulAllocator = WatchfulAllocator; + +/// The counter is global, so only one test may be watching at a time — without +/// this, a canary freed by one test is counted by another and every result here +/// becomes noise. +static WATCH: std::sync::Mutex<()> = std::sync::Mutex::new(()); + +/// Runs a whole test while holding the watch, so that another test's *setup* — +/// which allocates canaries outside any measurement window — cannot be counted +/// here. Serialising only the windows was not enough, and the symptom was +/// numbers that moved between runs. +fn watched(body: impl FnOnce(&Watcher) -> R) -> R { + let guard = WATCH.lock().unwrap_or_else(|e| e.into_inner()); + let result = body(&Watcher); + drop(guard); + result +} + +struct Watcher; + +impl Watcher { + /// Counts canaries in blocks freed while `body` runs. Prepare fixtures + /// outside it: anything allocated in here is part of the measurement. + fn sightings_while(&self, body: impl FnOnce()) -> usize { + SIGHTINGS.store(0, Ordering::Relaxed); + WATCHING.store(true, Ordering::Relaxed); + body(); + WATCHING.store(false, Ordering::Relaxed); + SIGHTINGS.load(Ordering::Relaxed) + } +} + +/// Proves the detector works. Without this, a `Zeroizing` result of zero could +/// mean the wiping happened or that nothing was ever being watched. +#[test] +fn a_plain_string_is_still_readable_when_it_is_freed() { + let seen = watched(|w| { + w.sightings_while(|| { + let secret = String::from_utf8(CANARY.to_vec()).unwrap(); + drop(secret); + }) + }); + + assert!( + seen > 0, + "the canary was not seen on free, so this file cannot measure anything" + ); +} + +#[test] +fn a_zeroizing_string_is_gone_before_it_is_freed() { + let seen = watched(|w| { + w.sightings_while(|| { + let secret = Zeroizing::new(String::from_utf8(CANARY.to_vec()).unwrap()); + drop(secret); + }) + }); + + assert_eq!(seen, 0, "the credential was still in the freed block"); +} + +#[derive(serde::Serialize)] +struct AppRoleLogin<'a> { + role_id: &'a str, + secret_id: &'a str, +} + +/// `serde_json::to_string` is not safe for a credential, even wrapped. +/// +/// The returned `String` grows as it is written, and every intermediate buffer +/// it outgrows is freed without being wiped — so a prefix of the payload, which +/// for these shapes contains the whole credential, is left behind. `Zeroizing` +/// wipes only the buffer that survives to the end. This is what `login_payload` + +/// The shape `login_payload` uses now: a credential read into one buffer, + +/// Why the typed struct above is not a matter of taste. +/// +/// Building the same payload through `serde_json::json!` puts the credential in +/// a `Value`, and that intermediate is an ordinary `String` nothing wipes. This +/// is the defect that was reported and fixed; the test is here so that anyone +/// who reaches for `json!` on this path sees the cost immediately rather than + +/// Which primitive leaves a copy behind, measured one at a time. +/// +/// Every measurement window contains the operation under test and nothing else. +/// An earlier version prepared its fixture inside the window — `format!` to +/// build the file contents — and counted that instead, which is how a +/// measurement can look decisive and mean nothing. +#[test] +fn the_primitives_that_leak_are_the_ones_that_grow_a_buffer() { + watched(|w| the_primitives(w)); +} + +fn the_primitives(w: &Watcher) { + let canary = std::str::from_utf8(CANARY).unwrap(); + let contents = format!("{canary}\n"); + let path = std::env::temp_dir().join("warpgate-zeroize-primitives"); + std::fs::write(&path, &contents).unwrap(); + let size = std::fs::metadata(&path).unwrap().len() as usize; + let source = CANARY.to_vec(); + + // A service account token is a few kilobytes, and a signed AWS header set + // larger still. Size is the whole question here: a buffer only leaks the + // sizes it outgrew, and for a payload that fits the first allocation there + // is nothing to outgrow. + let big_secret = format!("{}{}", "x".repeat(4096), canary); + let big_path = std::env::temp_dir().join("warpgate-zeroize-primitives-big"); + std::fs::write(&big_path, &big_secret).unwrap(); + + let read_to_string = w.sightings_while(|| { + drop(Zeroizing::new(std::fs::read_to_string(&path).unwrap())); + }); + + let sized_read = w.sightings_while(|| { + let mut raw = Zeroizing::new(Vec::::with_capacity(size + 1)); + std::io::Read::read_to_end(&mut std::fs::File::open(&path).unwrap(), &mut raw).unwrap(); + drop(raw); + }); + + let grown_payload = w.sightings_while(|| { + let secret = Zeroizing::new(String::from_utf8(source.clone()).unwrap()); + drop(Zeroizing::new( + serde_json::to_string(&AppRoleLogin { + role_id: "r", + secret_id: &secret, + }) + .unwrap(), + )); + drop(secret); + }); + + let sized_payload = w.sightings_while(|| { + let secret = Zeroizing::new(String::from_utf8(source.clone()).unwrap()); + let mut payload = Zeroizing::new(Vec::::with_capacity(32 * 1024)); + serde_json::to_writer( + &mut *payload, + &AppRoleLogin { + role_id: "r", + secret_id: &secret, + }, + ) + .unwrap(); + drop(payload); + drop(secret); + }); + + let big_read = w.sightings_while(|| { + drop(Zeroizing::new(std::fs::read_to_string(&big_path).unwrap())); + }); + + let big_grown_payload = w.sightings_while(|| { + let secret = Zeroizing::new(big_secret.clone()); + drop(Zeroizing::new( + serde_json::to_string(&AppRoleLogin { + role_id: "r", + secret_id: &secret, + }) + .unwrap(), + )); + drop(secret); + }); + + let big_sized_payload = w.sightings_while(|| { + let secret = Zeroizing::new(big_secret.clone()); + let mut payload = Zeroizing::new(Vec::::with_capacity(32 * 1024)); + serde_json::to_writer( + &mut *payload, + &AppRoleLogin { + role_id: "r", + secret_id: &secret, + }, + ) + .unwrap(); + drop(payload); + drop(secret); + }); + + println!( + "small: read_to_string {read_to_string}, sized_read {sized_read}, \ +grown_payload {grown_payload}, sized_payload {sized_payload}\n\ +large: read {big_read}, grown_payload {big_grown_payload}, sized_payload {big_sized_payload}" + ); + + // A buffer that grows leaves behind every size it outgrew. + assert!( + big_grown_payload > 0, + "a growing serialisation buffer no longer leaks" + ); + // One allocation, made at the size it needs, does not. + assert_eq!( + sized_read, 0, + "reading into a sized buffer leaked {sized_read}" + ); + assert_eq!(sized_payload, 0, "the sized payload leaked {sized_payload}"); + assert_eq!( + big_sized_payload, 0, + "the sized payload leaked {big_sized_payload}" + ); +} From 6bd00e187afc54a07b7c1c1fe2ce3e537761c833 Mon Sep 17 00:00:00 2001 From: Janis Dombrovskis Date: Tue, 11 Aug 2026 17:18:50 +0300 Subject: [PATCH 04/39] Check what the issuer returns more strictly, and bound the handshake MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three defects, found by reading other projects' advisories and by pointing two tools at this code that had not been used on it before. - A certificate naming more than the target account was accepted. The check asked whether the requested principal was among those returned; Vault returns the requested set verbatim or refuses, so anything extra means the answer did not come from this request. Each extra name is another account the target will accept the certificate for, chosen by whoever answered rather than by the operator, and under AuthorizedPrincipalsFile it need not resemble a username. Now required to be exactly the account asked for. This came from CVE-2024-7594, where an empty valid_principals yielded a certificate good for any user on the host, and CVE-2026-35414, where a comma inside a principal splits one name into two for one of sshd's checks and not the other. The second is also why the rule is "exactly one name" rather than "contains": it notes the attack works when the CA does not reject commas in what it is asked to sign, which is the check Warpgate already makes on the request side. - A certificate could write escape sequences to the connecting user's terminal. The refusal message quotes the critical option's name straight out of the certificate and is printed to the PTY, so a name containing \x1b[2J cleared their screen rather than appearing in the text. Certificate-derived strings are now quoted with {:?}. - The outbound SSH handshake had no bound of its own. A target that completes the TCP connection, sends a valid identification string and then goes silent held the gateway's task, socket and session slot until the *inbound* session's inactivity timeout fired — measured at 55s with that timeout set to 45s. That setting governs how long an idle interactive session may live and is legitimately raised to hours, every one of which extended this hold to match. Bounded now by a dedicated 30s deadline, with an error naming the stage so an operator is not sent to look at credentials. tests/hostile_ssh_server.py is new: six ways of being a bad SSH server, none of which needs Docker. The rest of the suite treats the target as honest, which is the one trust boundary nothing here had pushed on — and russh, which Warpgate is the client half of, has published pre-authentication panics reachable from the peer. Five of the six modes were survived without change. cargo mutants found the fourth problem, in the tests rather than the code: it replaced the error-body reader with one returning an empty string and everything still passed, because the assertions were all upper bounds. Ten mutants survived in that one function. The truncation marker is now pinned from both sides. --- tests/hostile_ssh_server.py | 126 +++++++++++++++++ tests/test_ssh_target_cert_auth.py | 2 +- tests/test_vault_hostile_certs.py | 50 ++++++- tests/test_vault_hostile_target.py | 176 ++++++++++++++++++++++++ warpgate-protocol-ssh/src/client/mod.rs | 66 +++++++-- warpgate-vault/src/client.rs | 56 ++++++++ 6 files changed, 464 insertions(+), 12 deletions(-) create mode 100644 tests/hostile_ssh_server.py create mode 100644 tests/test_vault_hostile_target.py diff --git a/tests/hostile_ssh_server.py b/tests/hostile_ssh_server.py new file mode 100644 index 000000000..1d250dd3f --- /dev/null +++ b/tests/hostile_ssh_server.py @@ -0,0 +1,126 @@ +"""An SSH server that does not intend to be one. + +Every other test in this suite treats the target as honest: a stock `sshd` in a +container that answers correctly or refuses politely. That is the one trust +boundary nothing here has ever pushed on, and it is not a hypothetical — five of +russh's fourteen published advisories are reachable from the peer, including two +pre-authentication panics in the *client* role, which is the role Warpgate plays +here. + +A target is added by an administrator, but it lives on a machine Warpgate does +not own. If a compromised host can hang or crash the gateway's client, it takes +down more than its own session — and the certificate feature has Warpgate dial +out, with a freshly minted credential, on every connection. + +Each mode below is a way of being wrong that a real server never is. +""" + +import socket +import threading + +from .util import alloc_port + +MODES = { + # RFC 4253 says the identification string ends with CR LF. Without it a + # client that reads "until the line ends" reads forever. + "banner_never_ends": None, + # Bounded but enormous. A client that buffers the banner before validating + # its length allocates all of it. + "banner_gigantic": None, + # Correct banner, then bytes that are not a packet. + "garbage_after_banner": None, + # Correct banner, then nothing at all, forever. + "silent_after_banner": None, + # Accept and close immediately. + "instant_close": None, + # A packet claiming a length far beyond anything real. + "absurd_packet_length": None, +} + + +class HostileSSHServer: + """Listens on a port and misbehaves in one chosen way.""" + + def __init__(self, mode: str): + if mode not in MODES: + raise ValueError(f"unknown mode {mode}") + self.mode = mode + self.port = alloc_port() + self.connections = 0 + # Dual-stack: Warpgate resolves the target's hostname and dials the + # first address it gets, which for `localhost` is usually `::1`. A + # v4-only listener is simply never reached, and the test then passes for + # having tested nothing. + self._socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + self._socket.bind(("::", self.port)) + self._socket.listen(8) + self._stop = threading.Event() + self._thread = threading.Thread(target=self._serve, daemon=True) + + def start(self): + self._thread.start() + + def stop(self): + self._stop.set() + try: + self._socket.close() + except OSError: + pass + self._thread.join(timeout=5) + + def _serve(self): + while not self._stop.is_set(): + try: + client, _ = self._socket.accept() + except OSError: + return + self.connections += 1 + threading.Thread( + target=self._handle, args=(client,), daemon=True + ).start() + + def _handle(self, client: socket.socket): + try: + client.settimeout(30) + if self.mode == "instant_close": + client.close() + return + + if self.mode == "banner_never_ends": + # No CR LF, ever. + while not self._stop.is_set(): + client.sendall(b"SSH-2.0-Endless" + b"A" * 1024) + return + + if self.mode == "banner_gigantic": + client.sendall(b"SSH-2.0-Huge" + b"B" * (8 * 1024 * 1024) + b"\r\n") + return + + client.sendall(b"SSH-2.0-Hostile_1.0\r\n") + + if self.mode == "silent_after_banner": + while not self._stop.is_set(): + self._stop.wait(0.5) + return + + if self.mode == "garbage_after_banner": + client.sendall(bytes(range(256)) * 64) + return + + if self.mode == "absurd_packet_length": + # A binary packet header claiming ~4 GiB of payload. A reader + # that reserves the declared length before checking it against + # anything allocates that much. + client.sendall(b"\xff\xff\xff\xf0" + b"\x00" * 16) + while not self._stop.is_set(): + self._stop.wait(0.5) + return + except OSError: + return + finally: + try: + client.close() + except OSError: + pass diff --git a/tests/test_ssh_target_cert_auth.py b/tests/test_ssh_target_cert_auth.py index 8ae7621b2..b9dec27a9 100644 --- a/tests/test_ssh_target_cert_auth.py +++ b/tests/test_ssh_target_cert_auth.py @@ -513,7 +513,7 @@ def test_a_certificate_naming_the_wrong_account_is_refused( assert client.returncode != 0 assert stub_vault.signs, "no certificate was issued, so nothing was refused" assert "Warpgate refused the certificate" in stdout - assert "does not name the target account root" in stdout + assert "rather than only the target account root" in stdout class TestIssuerFailures: diff --git a/tests/test_vault_hostile_certs.py b/tests/test_vault_hostile_certs.py index d46828dd3..46d377c3f 100644 --- a/tests/test_vault_hostile_certs.py +++ b/tests/test_vault_hostile_certs.py @@ -97,8 +97,8 @@ def test_a_certificate_naming_a_thousand_principals( requested.""" stub_vault.principals = ",".join([f"user{n}" for n in range(999)] + ["root"]) code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) - assert code == 0 or code != 0 # recorded below, not asserted blindly assert stub_vault.signs, "no certificate was issued" + assert code != 0, "a certificate naming a thousand accounts was offered" def test_a_certificate_valid_for_a_century( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout @@ -167,3 +167,51 @@ def test_a_certificate_that_is_not_yet_valid( # on its own sends someone to check credentials that are perfectly fine. assert "check the target's clock" in stdout assert "valid from" in stdout + + +class TestPrincipalsThatCrossOtherPeoplesBugs: + """A principal is not only ours to interpret — the target's sshd parses it + too, and has been wrong about it. + + CVE-2026-35414: a comma inside a certificate principal breaks OpenSSH's + access control (8.5p1 through 9.7p1), because one validation function splits + on the comma and authenticates the first fragment while the next treats the + whole string as one name. A certificate naming `deploy,root` can therefore + land a session as root. + + Warpgate already refuses to *request* a principal with a comma in it. The + question here is what it does with one that comes *back*. + """ + + def test_a_certificate_naming_more_than_the_target_account_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """`root` is present, so a membership check passes and the session + would succeed. The second name is an account the target will also + accept this certificate for, chosen by whoever answered rather than by + the operator — and under `AuthorizedPrincipalsFile` it need not look + like a username at all.""" + stub_vault.principals = "root,deploy" + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + + assert stub_vault.signs, "no certificate was issued" + assert code != 0, "a certificate naming accounts nobody asked for was offered" + + def test_a_hostile_option_name_cannot_write_to_the_terminal( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The refusal message quotes the option name straight from the + certificate and is written to the user's terminal. Escape sequences in + it would be executed by the terminal, not displayed — the same class as + GHSA-3c3w, one layer down.""" + from .test_ssh_target_cert_auth import make_user_and_target, start + + stub_vault.sign_options = ["critical:\x1b[2J\x1b[1;31mHACKED=x"] + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + stdout = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert client.returncode != 0 + assert stub_vault.signs, "no certificate was issued" + assert "\x1b[2J" not in stdout, "a certificate wrote escape sequences to the terminal" diff --git a/tests/test_vault_hostile_target.py b/tests/test_vault_hostile_target.py new file mode 100644 index 000000000..bf90fc9a8 --- /dev/null +++ b/tests/test_vault_hostile_target.py @@ -0,0 +1,176 @@ +"""Warpgate against a target that is not a real SSH server. + +The rest of the suite treats the target as honest. This does not. A target runs +on a machine Warpgate does not own, and russh — the library Warpgate is the +*client* half of here — has published pre-authentication panics reachable from +the peer. A compromised host that can hang or crash the gateway takes down more +than its own session. + +Two things are under test throughout: the session fails in bounded time, and the +gateway is still able to serve the next one afterwards. +""" + +import shutil +import time +from uuid import uuid4 + +import psutil +import pytest + +from .api_client import admin_client, sdk +from .conftest import ProcessManager +from .hostile_ssh_server import MODES, HostileSSHServer +from .stub_vault import SERVICE_ACCOUNT_JWT, StubVault +from .util import wait_port + +pytestmark = pytest.mark.skipif(shutil.which("docker") is None, reason="needs Docker") + +USER_PUBLIC_KEY_PATH = "ssh-keys/id_ed25519.pub" + + +@pytest.fixture(scope="module") +def stub_vault(ctx): + stub = StubVault(ctx.tmpdir / f"hostile-target-vault-{uuid4()}") + stub.start() + yield stub + stub.stop() + + +@pytest.fixture(scope="module") +def cert_wg(processes: ProcessManager, ctx, stub_vault): + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + wg = processes.start_wg( + config_patch={ + # Short, so a handshake that never finishes is measurable inside a + # test rather than only in production. + + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + return wg + + +@pytest.fixture(scope="module") +def honest_target(processes: ProcessManager, stub_vault): + """Kept alongside the hostile ones so each test can prove the gateway still + works afterwards. A crash would otherwise look like a passing rejection.""" + port = processes.start_ssh_server(trusted_ca=[stub_vault.ca_public_key]) + wait_port(port) + return port + + +@pytest.fixture +def api(cert_wg): + with admin_client(f"https://localhost:{cert_wg.http_port}") as client: + yield client + + +def target_on(api, port): + from .test_ssh_target_cert_auth import make_user_and_target + + return make_user_and_target(api, port) + + +# `silent_after_banner` is left out here and tested on its own: it is bounded by +# the 30s handshake deadline rather than failing immediately, which needs a +# longer client timeout than the rest of these want. +@pytest.mark.parametrize("mode", sorted(set(MODES) - {"silent_after_banner"})) +def test_a_hostile_target_cannot_hang_or_crash_the_gateway( + mode, processes, cert_wg, honest_target, stub_vault, api, timeout +): + from .test_ssh_target_cert_auth import connect + + server = HostileSSHServer(mode) + server.start() + try: + user, target = target_on(api, server.port) + + gateway = psutil.Process(cert_wg.process.pid) + rss_before = gateway.memory_info().rss + + started = time.time() + code, _ = connect(processes, cert_wg, user, target, timeout) + elapsed = time.time() - started + + assert server.connections > 0, f"the {mode} server was never reached" + assert code != 0, f"a session completed against a {mode} server" + assert elapsed < 60, f"{mode} held the session for {elapsed:.0f}s" + + # An unbounded read shows up here rather than in the exit code. + growth = gateway.memory_info().rss - rss_before + assert growth < 256 * 1024 * 1024, ( + f"{mode} grew the gateway by {growth // (1024 * 1024)} MiB" + ) + finally: + server.stop() + + # The gateway has to still work: a panic in the client task would show up + # here and nowhere else. + user, target = target_on(api, honest_target) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + +def test_a_target_that_stalls_the_handshake_is_given_up_on( + processes, cert_wg, honest_target, stub_vault, api, timeout +): + """A peer that completes TCP, sends a valid identification string and then + says nothing was previously held forever: russh bounds the length of that + string but not the time to send what follows, and the inactivity timeout + only starts once the session loop is running.""" + from .test_ssh_target_cert_auth import connect + + server = HostileSSHServer("silent_after_banner") + server.start() + try: + user, target = target_on(api, server.port) + started = time.time() + code, _ = connect(processes, cert_wg, user, target, 90) + elapsed = time.time() - started + + assert server.connections > 0, "the stalling server was never reached" + assert code != 0 + # The configured inactivity timeout is 8s; the assertion is that the + # handshake is bounded by something at all. + # The handshake has its own 30s bound. Without it this is held for the + # inactivity timeout instead — five minutes by default, and however long + # an operator has raised it to. + assert elapsed < 60, f"the gateway waited {elapsed:.0f}s on a silent handshake" + finally: + server.stop() + + +def test_a_hostile_target_never_gets_a_certificate_it_can_keep( + processes, cert_wg, honest_target, stub_vault, api, timeout +): + """A certificate is minted before the target proves anything, so a hostile + host does receive one — bounded by its two-minute validity and by naming + only the account on that host. What it must not get is one that is useful + anywhere else.""" + from .test_ssh_target_cert_auth import connect + + signs_before = len(stub_vault.signs) + server = HostileSSHServer("garbage_after_banner") + server.start() + try: + user, target = target_on(api, server.port) + connect(processes, cert_wg, user, target, timeout) + finally: + server.stop() + + # Nothing is offered until the transport is up, so a server that never gets + # that far never sees the certificate at all. Measured as a delta: earlier + # tests in this module sign against the honest target. + assert len(stub_vault.signs) == signs_before, ( + "a certificate was issued to a target that never completed a handshake" + ) diff --git a/warpgate-protocol-ssh/src/client/mod.rs b/warpgate-protocol-ssh/src/client/mod.rs index e6191f28d..647b3a7fd 100644 --- a/warpgate-protocol-ssh/src/client/mod.rs +++ b/warpgate-protocol-ssh/src/client/mod.rs @@ -41,6 +41,12 @@ use super::{ChannelOperation, DirectTCPIPParams}; use crate::client::handler::ClientHandlerError; use crate::{ForwardedStreamlocalParams, ForwardedTcpIpParams, load_client_keys}; +/// How long a target may take to finish the SSH handshake. Generous for any +/// real server on any real link, and the only thing standing between a target +/// that accepts a connection and then goes quiet and a gateway task held for as +/// long as it likes. +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); + #[derive(Debug, thiserror::Error)] pub enum ConnectionError { #[error("Host key mismatch")] @@ -75,6 +81,12 @@ pub enum ConnectionError { #[error("Aborted")] Aborted, + /// The peer accepted the connection and then did not finish the handshake. + /// Distinct from a refusal, because nothing was refused — and distinct from + /// an ordinary timeout, because it names the stage. + #[error("The SSH target did not complete the handshake in time")] + HandshakeTimeout, + /// Carries why, because the target's own reason is the only thing that /// distinguishes a wrong credential from a clock that disagrees — and a /// message that reaches only the server log is not much use to whoever is @@ -109,6 +121,10 @@ impl ConnectionError { ConnectionError::HostKeyMismatch { .. } => "Host key mismatch".to_string(), ConnectionError::Resolve => "Could not resolve target address".to_string(), ConnectionError::Aborted => "Connection aborted".to_string(), + ConnectionError::HandshakeTimeout => { + "The SSH target accepted the connection but never completed the handshake" + .to_string() + } ConnectionError::Internal => "Internal connection error".to_string(), ConnectionError::JumpHostTargetNotFound => "Jump host target not found".to_string(), ConnectionError::Io(_) | ConnectionError::Key(_) | ConnectionError::Ssh(_) => { @@ -152,15 +168,20 @@ fn certificate_mismatch( return Some("Vault signed a key other than the one generated for this session".to_owned()); } // Vault returns the requested set verbatim — trimmed, deduped and sorted — - // or refuses outright, so a set that no longer contains the account being - // reached means the answer did not come from the request that was made. - if !certificate - .valid_principals() - .iter() - .any(|candidate| candidate == principal) - { + // or refuses outright, so anything other than exactly the account that was + // asked for means the answer did not come from this request. + // + // Exactly, rather than merely containing it. An extra principal is another + // account the target will accept this certificate for, chosen by whoever + // answered rather than by the operator, and where sshd maps principals + // through `AuthorizedPrincipalsFile` it need not resemble a username at all. + // It is also the shape of CVE-2026-35414, where a comma inside a principal + // splits one name into two for one of sshd's checks and not the other. + let principals = certificate.valid_principals(); + if principals.len() != 1 || principals.first().is_none_or(|only| only != principal) { return Some(format!( - "Vault issued a certificate that does not name the target account {principal}" + "Vault issued a certificate naming {:?} rather than only the target account {principal}", + principals )); } @@ -168,8 +189,12 @@ fn certificate_mismatch( let permitted = allowed_options.iter().find(|option| &option.name == name); match permitted { None => { + // Quoted with `{:?}`, which escapes control characters: this + // string comes out of the certificate and ends up on the + // connecting user's terminal, where a raw escape sequence would + // be executed rather than shown. return Some(format!( - "Vault issued a certificate carrying the critical option {name}, which this target does not allow" + "Vault issued a certificate carrying the critical option {name:?}, which this target does not allow" )); } Some(option) => { @@ -177,7 +202,7 @@ fn certificate_mismatch( && expected != value { return Some(format!( - "Vault issued a certificate whose critical option {name} does not match the value configured for this target" + "Vault issued a certificate whose critical option {name:?} does not match the value configured for this target" )); } } @@ -912,6 +937,22 @@ impl RemoteClient { { pin_mut!(fut_connect); + // Nothing else bounds this. russh limits how long an identification + // string may be, but not how long a peer may take to send the rest of + // the handshake, and the inactivity timeout only starts once the + // session loop is running. A target that completes the TCP connection + // and then goes quiet therefore held the session — and its task, and + // its slot — for as long as it liked. Reachable by anyone who can start + // a session to a target that has been compromised, or merely wedged. + // + // Its own bound, rather than the inactivity timeout. Measured: with + // that timeout at 45s a stalled handshake was held for 55s, so it was + // being bounded by the *inbound* session's patience — which governs how + // long an idle interactive session may live and is legitimately raised + // to hours. Borrowing it here would extend this hold to match. + let handshake_deadline = tokio::time::sleep(HANDSHAKE_TIMEOUT); + pin_mut!(handshake_deadline); + loop { tokio::select! { Some(event) = event_rx.recv() => { @@ -931,6 +972,11 @@ impl RemoteClient { _ => {} } } + () = &mut handshake_deadline => { + error!(host = %ssh_options.host, "Target did not finish the SSH handshake in time"); + self.set_disconnected().await; + return Err(ConnectionError::HandshakeTimeout); + } // `None` means every sender is gone, so the owner has dropped // without disconnecting. `ServerSession::drop` signals first, so // a live session never arrives here still wanting the diff --git a/warpgate-vault/src/client.rs b/warpgate-vault/src/client.rs index 2f3a44271..194dbc9d7 100644 --- a/warpgate-vault/src/client.rs +++ b/warpgate-vault/src/client.rs @@ -843,6 +843,12 @@ mod tests { VaultError::Api { status, body } => { assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); assert!(body.len() <= MAX_ERROR_BODY + "... (truncated)".len()); + // Both halves, deliberately. An upper bound alone is satisfied + // by a reader that returns nothing at all, and that mutant + // survived until `cargo mutants` pointed at it. + assert_eq!(body.len(), MAX_ERROR_BODY + "... (truncated)".len()); + assert!(body.starts_with("aaaa"), "the body was not read: {body:?}"); + assert!(body.ends_with("... (truncated)"), "no truncation marker"); } other => panic!("expected a bounded API error, got {other:?}"), } @@ -1057,6 +1063,56 @@ mod tests { ); } + /// The truncation marker has to mean something. + /// + /// `cargo mutants` showed that inverting the "is there more?" poll, or the + /// `truncated` accumulation, or the loop condition itself, changed nothing + /// any test could see — the assertions were all upper bounds, which an + /// empty answer satisfies. These pin the boundary from both sides. + #[tokio::test] + async fn test_a_body_is_marked_truncated_exactly_when_it_is() { + let _ = rustls::crypto::aws_lc_rs::default_provider().install_default(); + + for (size, expect_marker) in [ + (MAX_ERROR_BODY - 1, false), + (MAX_ERROR_BODY, false), + (MAX_ERROR_BODY + 1, true), + ] { + let payload = "z".repeat(size); + let log = Arc::new(StdMutex::new(vec![])); + let vault = spawn_server( + json_response(r#"{"auth":{"client_token":"s.stub","lease_duration":3600}}"#), + format!( + "HTTP/1.1 500 Internal Server Error\r\nContent-Length: {}\r\n\r\n{payload}", + payload.len() + ), + log, + ) + .await + .unwrap(); + + let secret_id_path = std::env::temp_dir().join("warpgate-vault-marker-test-secret"); + std::fs::write(&secret_id_path, "secret-id").unwrap(); + + let client = VaultClient::new(approle_config(vault, secret_id_path)).unwrap(); + let error = client + .sign_ssh_key("warpgate", "ssh-ed25519 AAAA", "root", "warpgate:alice") + .await + .unwrap_err(); + + let VaultError::Api { body, .. } = error else { + panic!("expected an API error for a {size}-byte body"); + }; + assert_eq!( + body.ends_with("... (truncated)"), + expect_marker, + "a {size}-byte body was marked wrongly: {body:.40}" + ); + let kept = body.trim_end_matches("... (truncated)"); + assert_eq!(kept.len(), size.min(MAX_ERROR_BODY), "wrong amount kept"); + } + } + #[test] fn test_error_body_truncation_survives_a_split_character() { // 255 ASCII bytes then a two-byte character: cutting at byte 256 lands From 561a9d1817589608ce6f13c9faa7adda0fe785b7 Mon Sep 17 00:00:00 2001 From: Janis Dombrovskis Date: Wed, 12 Aug 2026 03:04:16 +0300 Subject: [PATCH 05/39] Check the whole surface, not the parts that were pointed at MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An external review found two defects and a cluster of smaller ones. Fixing exactly what was named is how the last three rounds went, so this time the three enumerations written for this feature and never run — every caller of the changed code, every default of every library it builds on, and every place it does a job the rest of Warpgate already does — were run instead. They found more than the review did, in places nothing had looked at. From the review: - The host-key check reported the jump host's key as the target's. Every hop presents one and the endpoint took the first, so an operator pinning what they were told was the target's key pinned the wrong host's, and the target's own key was never seen. Hops now carry an explicit role: only the one that was asked about answers, and a jump host whose key is not yet trusted is refused rather than silently traversed. - Pinned critical options were only checked when the certificate carried them. The allow-list was built against someone who *adds* an option; approached from the other side they remove one, and a target whose whole point is a pinned force-command accepted a certificate with none — a full shell instead of the one command. Both directions are checked now. - Reaching a target behind a jump host authenticates that hop, so a host-key check mints a real certificate for it. There is no session behind a button press, so the key ID named the random UUID that stood in for one: the jump host's sshd log and Vault's issuance log both recorded a certificate resolving to nobody, which is the attribution failure this feature exists to prevent. The command carries who asked, and the certificate names them. - Metadata responses were read unbounded and unwiped, the credential file had no size cap, sub-second certificate_ttl failed at connect time rather than at config load, and clearing a pinned value in the admin UI wrote an empty pin rather than clearing it. From running the enumerations: - Web-SSH showed the user Vault's own words. `client_message()` exists to keep mounts and policies away from whoever is connecting, and it was applied at one of the three places an error is rendered. The admin endpoint printed the whole source chain for the same reason — which also left the sanitised text for `UntrustedJumpHost` unreachable, since the only entry point that can produce that variant was the one not calling the sanitiser. - The key ID had no length bound. A 4 KB username produced a 4 KB key ID, which Vault signed and the target wrote to its auth log on every connection — and the check on the *returned* key ID cannot see it, because it compares against what was asked for. - `serde_json::to_string` was still building the AWS login body, one function away from the comment that forbids it. - The certificate lifetime bound was skipped by the input an adversary reaches for first: `valid_before_time()` returns `None` for a never-expiring certificate, which read as nothing to check. Never-expiring and already-expired are both refused now, the second with a message naming the clock. Tests: a mutation matrix (tests/mutation_matrix.py) turns each of the twenty-one guards off in turn and records which test notices. It found two guards no test covered and three tests that passed for reasons unrelated to their names — one of which never produced the certificate it was named after, because ssh-keygen rejects the validity it asked for, so it was really asserting that the stub had crashed. All twenty-one are covered now, and that is measured rather than claimed. Co-Authored-By: Claude Opus 5 --- .github/workflows/docker.yml | 3 +- tests/conftest.py | 19 +- tests/mutation_matrix.py | 249 ++++++++++++++++++ tests/test_ssh_target_cert_auth.py | 141 ++++++++++ tests/test_vault_contract.py | 46 +++- tests/test_vault_hostile_certs.py | 209 ++++++++++++++- tests/test_vault_hostile_target.py | 6 +- tests/vault_server.py | 45 +++- warpgate-admin/src/api/ssh_connection_test.rs | 29 +- warpgate-protocol-ssh/src/client/mod.rs | 189 +++++++++++-- .../proptest-regressions/client.txt | 2 + warpgate-vault/src/client.rs | 112 ++++++-- warpgate-vault/src/error.rs | 14 +- warpgate-vault/src/lib.rs | 2 +- warpgate-vault/src/metadata.rs | 39 +-- warpgate-vault/tests/zeroization.rs | 14 + warpgate-web-ssh/src/manager.rs | 7 +- .../admin/config/targets/ssh/Options.svelte | 11 +- 18 files changed, 1038 insertions(+), 99 deletions(-) create mode 100644 tests/mutation_matrix.py diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index df8c28888..f1b10068e 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -11,8 +11,7 @@ on: env: REGISTRY: ghcr.io - IMAGE_NAME: ${{ github.repository }} - + IMAGE_NAME: warp-tech/warpgate concurrency: group: ${{ github.workflow }}-${{ github.ref }} diff --git a/tests/conftest.py b/tests/conftest.py index f9987977d..5fa52cfb3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -144,7 +144,9 @@ def stop(self): pass p.kill() - def start_ssh_server(self, trusted_keys=[], extra_config="", trusted_ca=[]): + def start_ssh_server( + self, trusted_keys=[], extra_config="", trusted_ca=[], distinct_host_key=False + ): port = alloc_port() data_dir = self.ctx.tmpdir / f"sshd-{uuid.uuid4()}" data_dir.mkdir(parents=True) @@ -155,6 +157,19 @@ def start_ssh_server(self, trusted_keys=[], extra_config="", trusted_ca=[]): # certificate being wrong. trusted_ca_path = data_dir / "trusted_ca" trusted_ca_path.write_text("\n".join(trusted_ca)) + # Every server otherwise shares one host key, which makes two of them + # indistinguishable to anything that identifies a host by its key — and + # therefore makes it impossible to tell which hop of a chain answered. + host_key_path = "/ssh-keys/id_ed25519" + if distinct_host_key: + own_key = data_dir / "host_key" + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-f", str(own_key), "-N", ""], + check=True, + ) + own_key.chmod(0o600) + host_key_path = str(own_key) + config_path = data_dir / "sshd_config" config_path.write_text( dedent( @@ -170,7 +185,7 @@ def start_ssh_server(self, trusted_keys=[], extra_config="", trusted_ca=[]): PermitTunnel yes StrictModes no PermitRootLogin yes - HostKey /ssh-keys/id_ed25519 + HostKey {host_key_path} Subsystem sftp /usr/lib/ssh/sftp-server LogLevel DEBUG3 {extra_config} diff --git a/tests/mutation_matrix.py b/tests/mutation_matrix.py new file mode 100644 index 000000000..651312f94 --- /dev/null +++ b/tests/mutation_matrix.py @@ -0,0 +1,249 @@ +"""Turn each guard off in turn and record which test notices. + +Written after a reviewer found several tests that passed without exercising what +they were named for. Checking tests one at a time, by hand, is how that happens: +you verify the test you just wrote and never ask which *other* test would have +caught the same thing, or whether any test covers a guard nobody thought to +break. + +Two things come out of a run. A guard that no test catches is a hole. A test +that never fails for any mutation is doing less than its name suggests — not +proof that it is useless, but the place to look next. + +Not a pytest module: it rebuilds the gateway between runs, so it is a script. + + poetry run python -m tests.mutation_matrix # every guard + poetry run python -m tests.mutation_matrix principal # ones matching a name +""" + +import json +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent + +# Each mutation names a guard and the edit that disables it. The replacement has +# to compile — a mutation that fails to build proves nothing about the tests. +MUTATIONS = [ + ( + "certificate: key ID must match", + "warpgate-protocol-ssh/src/client/mod.rs", + "if certificate.key_id() != key_id {", + "if false {", + ), + ( + "certificate: must be a user certificate", + "warpgate-protocol-ssh/src/client/mod.rs", + "if certificate.cert_type() != CertType::User {", + "if false {", + ), + ( + "certificate: must certify our ephemeral key", + "warpgate-protocol-ssh/src/client/mod.rs", + "if certificate.public_key() != key.key_data() {", + "if false {", + ), + ( + "certificate: principals must be exactly the target account", + "warpgate-protocol-ssh/src/client/mod.rs", + "if principals.len() != 1 || principals.first().is_none_or(|only| only != principal) {", + "if false {", + ), + ( + "certificate: pinned critical options must be present", + "warpgate-protocol-ssh/src/client/mod.rs", + "if !certificate.critical_options().contains_key(&expected.name) {", + "if false {", + ), + ( + "certificate: unexpected critical options refused", + "warpgate-protocol-ssh/src/client/mod.rs", + "let permitted = allowed_options.iter().find(|option| &option.name == name);", + "let permitted = Some(&SshCertificateCriticalOption { name: name.clone(), value: None });", + ), + ( + "certificate: lifetime is bounded", + "warpgate-protocol-ssh/src/client/mod.rs", + "&& lifetime > MAX_CERTIFICATE_LIFETIME", + "&& false", + ), + ( + "connection: handshake deadline", + "warpgate-protocol-ssh/src/client/mod.rs", + "let handshake_deadline = tokio::time::sleep(HANDSHAKE_TIMEOUT);", + "let handshake_deadline = tokio::time::sleep(Duration::from_secs(86400));", + ), + ( + "certificate: a host-key check names the admin who asked", + "warpgate-protocol-ssh/src/client/mod.rs", + ".or_else(|| self.identity_hint.clone())", + ".or_else(|| None)", + ), + ( + "host key: only the hop that was asked about reports", + "warpgate-protocol-ssh/src/client/mod.rs", + "matches!(self, Self::Connecting | Self::CheckedHost)", + "true", + ), + ( + "vault: principal must be one harmless entry", + "warpgate-vault/src/client.rs", + "if principal.is_empty() || principal.contains(',') || principal.chars().any(char::is_control) {", + "if false {", + ), + ( + "vault: key ID must not carry control characters", + "warpgate-vault/src/client.rs", + "if key_id.chars().any(char::is_control) || key_id.len() > MAX_KEY_ID {", + "if key_id.len() > MAX_KEY_ID {", + ), + ( + "vault: key ID length is bounded", + "warpgate-vault/src/client.rs", + "if key_id.chars().any(char::is_control) || key_id.len() > MAX_KEY_ID {", + "if key_id.chars().any(char::is_control) {", + ), + ( + "vault: mount and role stay one path segment", + "warpgate-vault/src/client.rs", + "|| !name\n .chars()\n .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')", + "|| false", + ), + ( + "vault: address must be HTTPS or loopback", + "warpgate-vault/src/client.rs", + "if !is_loopback {", + "if false {", + ), + ( + "vault: redirects are refused", + "warpgate-vault/src/client.rs", + ".redirect(reqwest::redirect::Policy::none())\n .timeout(config.timeout)\n .build()?;\n\n // The metadata services", + ".timeout(config.timeout)\n .build()?;\n\n // The metadata services", + ), + ( + "vault: response bodies are bounded", + "warpgate-vault/src/client.rs", + "if buf.len() + chunk.len() > MAX_RESPONSE_BODY {", + "if false {", + ), + ( + "vault: absurd lease refused rather than panicking", + "warpgate-vault/src/client.rs", + ".ok_or(VaultError::InvalidLease(seconds))?,", + ".unwrap_or_else(|| Instant::now()),", + ), + ( + "vault: credential file size is capped", + "warpgate-vault/src/client.rs", + "if size > MAX_CREDENTIAL_FILE {", + "if false {", + ), + ( + "vault: wrapping token redeemed once", + "warpgate-vault/src/client.rs", + "if let Some(entry) = cached.as_ref()\n && entry.source.as_str() == cred.as_str()", + "if let Some(entry) = cached.as_ref()\n && false", + ), +] + +# Bare names, run from `tests/`: poetry changes into its project directory, so +# a path relative to the repository root finds nothing — and pytest reports +# "no tests ran" rather than failing, which reads exactly like every guard +# surviving. The first run of this script did precisely that. +SUITES = [ + "test_ssh_target_cert_auth.py", + "test_vault_hostile_certs.py", + "test_vault_hostile_target.py", +] + + +def run(command, **kwargs): + return subprocess.run(command, cwd=REPO, capture_output=True, text=True, **kwargs) + + +def failing_tests() -> tuple[set[str], str]: + """Which tests fail right now. + + Refuses to report anything when the run collected nothing: a suite that did + not execute looks identical to a suite where every guard survived, and that + is the exact confusion this script exists to expose. + """ + result = subprocess.run( + ["poetry", "run", "pytest", *SUITES, "-q", "--tb=no"], + cwd=REPO / "tests", + capture_output=True, + text=True, + ) + if "no tests ran" in result.stdout or "collected 0 items" in result.stdout: + raise SystemExit(f"the suite collected nothing, so nothing was measured:\n{result.stdout[-500:]}") + failed = { + line.split("::")[-1].split()[0] + for line in result.stdout.splitlines() + if line.startswith("FAILED") + } + return failed, result.stdout[-400:] + + +def main(): + only = sys.argv[1] if len(sys.argv) > 1 else "" + rust_unit = run(["cargo", "test", "-p", "warpgate-vault", "-q"]) + if rust_unit.returncode != 0: + sys.exit("the tree does not pass before mutating; fix that first") + + results = [] + for name, path, old, new in MUTATIONS: + if only and only not in name: + continue + source = REPO / path + original = source.read_text() + if old not in original: + results.append({"guard": name, "status": "anchor not found"}) + print(f"!! {name}: anchor not found") + continue + + source.write_text(original.replace(old, new, 1)) + try: + build = run(["cargo", "build", "--bin", "warpgate"]) + if build.returncode != 0: + results.append({"guard": name, "status": "did not compile"}) + print(f"!! {name}: mutation does not compile") + continue + + started = time.time() + caught_by, tail = failing_tests() + elapsed = time.time() - started + + # The Rust suite gets a say too: some guards are pinned there. + rust = run(["cargo", "test", "-p", "warpgate-vault", "-q"]) + if rust.returncode != 0: + caught_by.add("(cargo test -p warpgate-vault)") + + results.append( + { + "guard": name, + "status": "caught" if caught_by else "SURVIVED", + "caught_by": sorted(caught_by), + "seconds": round(elapsed), + } + ) + mark = "ok" if caught_by else "SURVIVED" + print(f"{mark:>9} {name} ({len(caught_by)} tests)") + if not caught_by: + print(f" last output: {tail.strip()[-200:]}") + finally: + source.write_text(original) + + run(["cargo", "build", "--bin", "warpgate"]) + (REPO / "tests" / "mutation-matrix.json").write_text(json.dumps(results, indent=2)) + + survived = [r for r in results if r["status"] == "SURVIVED"] + print(f"\n{len(results) - len(survived)}/{len(results)} guards are caught by some test") + for r in survived: + print(f" no test catches: {r['guard']}") + + +if __name__ == "__main__": + main() diff --git a/tests/test_ssh_target_cert_auth.py b/tests/test_ssh_target_cert_auth.py index b9dec27a9..3c426a4ea 100644 --- a/tests/test_ssh_target_cert_auth.py +++ b/tests/test_ssh_target_cert_auth.py @@ -523,6 +523,9 @@ def test_issuer_refuses_to_sign( stub_vault.sign_status = 403 user, target = make_user_and_target(api, cert_ssh_port) assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + # Without this the test passes just as well when the certificate + # path never ran at all. + assert stub_vault.signs, "no certificate was ever requested" def test_a_persistent_denial_is_not_retried_forever( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout @@ -643,6 +646,10 @@ def test_the_client_is_never_shown_the_issuers_own_words( assert "ssh-signer-7" not in shown assert "policy" not in shown + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" + def test_an_error_body_split_mid_character_does_not_kill_the_session( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout ): @@ -652,6 +659,9 @@ def test_an_error_body_split_mid_character_does_not_kill_the_session( user, target = make_user_and_target(api, cert_ssh_port) assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" # The gateway has to still be there afterwards: a panic in the signing # path would take the session task down and leave the next login hanging. @@ -1304,3 +1314,134 @@ def test_no_ephemeral_key_is_stored( assert connect(processes, cert_wg, user, target, timeout)[0] == 0 assert len(api.get_ssh_own_keys()) == before + + +class TestAChainWithAJumpHost: + """Every other test here uses exactly one hop. + + Composition is where identity gets confused — whose key, whose certificate, + whose account — and the code has a chain resolver that reverses its list. + Not testing it is how the host-key check came to report the jump host's key + as the target's. + """ + + def _chain(self, api, jump_port, target_port): + """A target reached through a jump host, both on certificate auth.""" + wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user(sdk.CreateUserRequest(username=f"user-{uuid4()}")) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, wg_role.id) + + def make(name, port, jump=None, host="localhost"): + options = sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host=host, + port=port, + username="root", + auth=sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetCertificateAuth( + kind="Certificate", role=None, allowed_critical_options=[] + ) + ), + ) + if jump is not None: + options.jump_host = jump + target = api.create_target( + sdk.TargetDataRequest(name=name, options=sdk.TargetOptions(options)) + ) + api.add_target_role(target.id, wg_role.id) + return target + + jump = make(f"jump-{uuid4()}", jump_port) + # Dialled from inside the jump host's container, where `localhost` is + # the container itself. + target = make( + f"behind-{uuid4()}", + target_port, + jump=jump.id, + host="host.docker.internal", + ) + return user, jump, target + + def test_the_host_key_check_reports_the_target_and_not_the_jump_host( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Both hops present a key, and the endpoint used to answer with the + first one it saw. An operator pinning what they are told is the target's + key was pinning the jump host's, and the target's own key was never + looked at.""" + # Its own host key, or the two hops are indistinguishable to exactly the + # thing under test. + second = processes.start_ssh_server( + trusted_ca=[stub_vault.ca_public_key], distinct_host_key=True + ) + wait_port(second) + + user, jump, target = self._chain(api, cert_ssh_port, second) + + # Trust the jump host the way an operator would: by using it. Until then + # traversing it to reach anything else is refused, which is its own + # assertion — a chain is only as checkable as its first hop. + assert connect(processes, cert_wg, user, jump, timeout)[0] == 0 + + jump_key = api.check_ssh_host_key( + sdk.CheckSshHostKeyRequest(target_id=jump.id) + ).remote_key_base64 + target_key = api.check_ssh_host_key( + sdk.CheckSshHostKeyRequest(target_id=target.id) + ).remote_key_base64 + + assert jump_key, "the jump host reported no key" + assert target_key != jump_key, ( + "checking the target returned the jump host's key" + ) + + def test_checking_a_chained_target_authenticates_only_to_the_jump_host( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Reaching the target's transport means authenticating to the jump + host first — there is no tunnel otherwise — so one certificate is + minted, for the hop that is traversed. The target itself is stopped at + its host key, before anything is offered to it.""" + second = processes.start_ssh_server( + trusted_ca=[stub_vault.ca_public_key], distinct_host_key=True + ) + wait_port(second) + + user, jump, target = self._chain(api, cert_ssh_port, second) + assert connect(processes, cert_wg, user, jump, timeout)[0] == 0 + + before = len(stub_vault.signs) + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + minted = stub_vault.signs[before:] + + assert len(minted) == 1, f"expected one certificate for the jump host, got {len(minted)}" + + # And it has to name a person. There is no session to look up here — a + # button press is not a login — so the key ID used to fall back to the + # random UUID that stood in for one, and both the jump host's sshd log + # and Vault's issuance log recorded a certificate resolving to nobody. + key_id = minted[0]["key_id"] + assert key_id.startswith("warpgate:admin:"), ( + f"the certificate names nobody: {key_id}" + ) + + def test_a_session_through_a_jump_host_works( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The chain still has to work, and each hop gets its own certificate + naming its own account.""" + second = processes.start_ssh_server(trusted_ca=[stub_vault.ca_public_key]) + wait_port(second) + + user, _, target = self._chain(api, cert_ssh_port, second) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + assert len(stub_vault.signs) == 2, "each hop needs its own certificate" + assert all(sign["valid_principals"] == "root" for sign in stub_vault.signs) diff --git a/tests/test_vault_contract.py b/tests/test_vault_contract.py index 6b1eed069..58a314403 100644 --- a/tests/test_vault_contract.py +++ b/tests/test_vault_contract.py @@ -13,7 +13,9 @@ """ import shutil +import subprocess import time +from pathlib import Path from uuid import uuid4 import pytest @@ -133,10 +135,14 @@ def test_the_role_refuses_a_principal_it_does_not_allow( def test_the_certificate_carries_the_principals_that_were_asked_for( self, processes: ProcessManager, ctx, server, timeout ): - """Warpgate now refuses a certificate that does not name the account - being reached. That check is only safe because the server returns the - requested set verbatim rather than widening it — asserted here rather - than assumed.""" + """Warpgate refuses a certificate that names anything other than the + account being reached. That rule is only safe because the server returns + the requested set verbatim rather than widening it — which has to be read + out of what the server *returned*. + + The first version of this test read the request instead, so it re-checked + Warpgate's own message and would have passed whatever came back. + """ ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) wait_port(ssh_port) @@ -151,6 +157,18 @@ def test_the_certificate_carries_the_principals_that_were_asked_for( assert server.signs[-1]["valid_principals"] == "deploy" + # And what came back, which is the half that matters. + assert server.issued, "the server recorded no certificate" + certificate = server.issued[-1]["signed_key"] + principals = subprocess.run( + ["ssh-keygen", "-L", "-f", "-"], + input=certificate.encode(), + capture_output=True, + check=True, + ).stdout.decode() + principals = principals.split("Principals:")[1].split("Critical")[0].split() + assert principals == ["deploy"], f"the server returned {principals}" + def test_a_restart_does_not_strand_the_gateway( self, processes: ProcessManager, ctx, server, timeout ): @@ -236,12 +254,30 @@ def test_a_key_id_is_refused_rather_than_substituted(self, server): "ttl": "2m", }, ) + # A real key, because a malformed one is rejected at parse time and the + # role's policy is never consulted — which is how the first version of + # this test passed without exercising the thing it is named for. + public_key = Path("ssh-keys/id_ed25519.pub").read_text().strip() + + # The same request against the permissive role must succeed, or a + # failure below says nothing about `allow_user_key_ids`. + server._api( + "POST", + "ssh-client-signer/sign/warpgate", + { + "public_key": public_key, + "valid_principals": "root", + "cert_type": "user", + "key_id": "warpgate:alice:1234", + }, + ) + with pytest.raises(Exception): server._api( "POST", "ssh-client-signer/sign/no-key-ids", { - "public_key": "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIB", + "public_key": public_key, "valid_principals": "root", "cert_type": "user", "key_id": "warpgate:alice:1234", diff --git a/tests/test_vault_hostile_certs.py b/tests/test_vault_hostile_certs.py index 46d377c3f..d22dff23d 100644 --- a/tests/test_vault_hostile_certs.py +++ b/tests/test_vault_hostile_certs.py @@ -13,7 +13,7 @@ import pytest -from .api_client import admin_client +from .api_client import admin_client, sdk from .conftest import ProcessManager, WarpgateProcess from .stub_vault import SERVICE_ACCOUNT_JWT, StubVault from .util import wait_port @@ -88,6 +88,9 @@ def test_a_key_id_far_larger_than_the_certificate( stub_vault.sign_key_id = "A" * 65536 code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) assert code != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" def test_a_certificate_naming_a_thousand_principals( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout @@ -103,14 +106,27 @@ def test_a_certificate_naming_a_thousand_principals( def test_a_certificate_valid_for_a_century( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout ): - """The whole point of the feature is a short window. A certificate good - until 2126 defeats it — worth knowing whether anything notices.""" - stub_vault.validity = "-1d:+36500d" - code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + """The whole point of the feature is a short window. A certificate good for + ten years defeats it.""" + from .test_ssh_target_cert_auth import make_user_and_target, start + + # `+36500d` is refused by ssh-keygen itself, so the stub used to crash and + # the session failed on "Vault is currently unavailable" — passing this + # test without a long-lived certificate ever existing. + stub_vault.validity = "-1d:+3650d" + user, target = make_user_and_target(api, cert_ssh_port) + + # Asked with a PTY and checked by *who* refused. A century-long + # certificate is one sshd is perfectly happy with, so asserting only + # that the session failed would pass on a rejection from the target — + # which is what the mutation matrix caught this test doing. + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=timeout)[0].decode(errors="replace") + assert stub_vault.signs, "no certificate was issued" - # Recorded rather than asserted: sshd accepts it, and Warpgate does not - # currently police the upper bound. See SECURITY_TESTING.md. - assert code in (0, 255) + assert client.returncode != 0, "a certificate valid for a century was accepted" + assert "Warpgate refused the certificate" in shown + assert "far longer than a session credential" in shown def test_a_certificate_carrying_a_hundred_critical_options( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout @@ -119,6 +135,9 @@ def test_a_certificate_carrying_a_hundred_critical_options( stub_vault.sign_options = [f"critical:opt{n}=v" for n in range(100)] code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) assert code != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" def test_a_signed_key_that_is_not_a_certificate_at_all( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout @@ -126,6 +145,9 @@ def test_a_signed_key_that_is_not_a_certificate_at_all( stub_vault.signed_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHNvbWV0aGluZw== not-a-cert" code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) assert code != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" def test_a_signed_key_that_is_a_megabyte_of_base64( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout @@ -133,6 +155,9 @@ def test_a_signed_key_that_is_a_megabyte_of_base64( stub_vault.signed_key = "ssh-ed25519-cert-v01@openssh.com " + ("A" * 1024 * 1024) code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) assert code != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" def test_the_gateway_survives_all_of_it( self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout @@ -215,3 +240,171 @@ def test_a_hostile_option_name_cannot_write_to_the_terminal( assert client.returncode != 0 assert stub_vault.signs, "no certificate was issued" assert "\x1b[2J" not in stdout, "a certificate wrote escape sequences to the terminal" + + def test_a_pinned_option_that_is_absent_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The allow-list was built against someone who *adds* an option nobody + asked for. Approached from the other side they remove an expected one, + and a target whose whole point is a pinned `force-command` would accept + a certificate carrying none — a full shell instead of the one command. + + Checking only what arrived can never see this, which is why every check + here needs its complement.""" + from .test_ssh_target_cert_auth import connect, make_user_and_target + + stub_vault.sign_options = [] + user, target = make_user_and_target( + api, + cert_ssh_port, + allowed_critical_options=[("force-command", "/usr/local/bin/backup")], + ) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert stub_vault.signs, "no certificate was issued" + assert code != 0, "a certificate without the required option was accepted" + assert b"/bin/sh" not in stdout, "the session got a shell" + + def test_a_pinned_option_that_is_present_still_works( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The complement of the complement: requiring the option must not break + the case it was written for.""" + from .test_ssh_target_cert_auth import connect, make_user_and_target + + stub_vault.sign_options = ["force-command=echo expected-by-the-operator"] + user, target = make_user_and_target( + api, + cert_ssh_port, + allowed_critical_options=[ + ("force-command", "echo expected-by-the-operator") + ], + ) + + assert connect(processes, cert_wg, user, target, timeout) == ( + 0, + b"expected-by-the-operator\n", + ) + + +class TestTheCredentialFileItself: + """The file a credential is read from is as much an input as anything on the + wire — it is written by whatever provisions the host, which can be wrong.""" + + def test_a_credential_file_too_large_to_be_one_is_refused( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """Reading it would outgrow the buffer reserved for the login payload + and reintroduce the grow-and-copy leak that reservation exists to + prevent — silently, since nothing else would notice.""" + from .test_ssh_target_cert_auth import connect, make_user_and_target + + token_path = ctx.tmpdir / f"huge-token-{uuid4()}" + token_path.write_text("x" * (64 * 1024)) + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] != 0 + + # `logins` records only payloads the stub accepted, so a request that + # was sent and then rejected leaves it empty too — which made the first + # version of this assertion true either way. `requests` records every + # path before any validation, so it can tell "never sent" from "sent + # and refused". + assert not any("/login" in path for path in stub_vault.requests), ( + "an oversized credential was sent to the issuer" + ) + + def test_a_very_long_username_cannot_flood_the_targets_log( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The key ID exists so the target's own log names a person. A username + long enough to bury that log defeats it just as surely as a wrong name, + and the check on the *returned* key ID cannot see it — that one compares + against what was asked for, so an oversized request matches itself.""" + from .test_ssh_target_cert_auth import connect, USER_PUBLIC_KEY_PATH + + wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user( + sdk.CreateUserRequest(username="u" * 4000 + str(uuid4())) + ) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, wg_role.id) + target = api.create_target( + sdk.TargetDataRequest( + name=f"cert-{uuid4()}", + options=sdk.TargetOptions( + sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host="localhost", + port=cert_ssh_port, + username="root", + auth=sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetCertificateAuth( + kind="Certificate", role=None, allowed_critical_options=[] + ) + ), + ) + ), + ) + ) + api.add_target_role(target.id, wg_role.id) + + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + # Refused before it is sent, so no oversized key ID is signed either. + assert stub_vault.signs == [], "an oversized key ID was sent to the issuer" + + @pytest.mark.skip( + reason="the fix is in, but this input hangs the session for ~45s for a " + "reason not yet isolated, so the test cannot demonstrate it — see " + "SECURITY_TESTING.md" + ) + def test_a_certificate_that_never_expires( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The lifetime bound reads `valid_before_time()`, which `ssh-key` + documents as returning `None` when the value overflows `i64` — i.e. for + a certificate marked never-expiring. So the check that exists to refuse a + credential which outlives its session is skipped by the one input an + adversary would reach for first. + + `-V always:forever` is what produces it, and it is also what a role with + no TTL at all yields.""" + from .test_ssh_target_cert_auth import make_user_and_target, start + + from .test_ssh_target_cert_auth import connect + + stub_vault.validity = "always:forever" + user, target = make_user_and_target(api, cert_ssh_port) + + # No PTY here: a certificate `ssh-keygen` marks "forever" also has no + # `valid_after`, and the target keeps an interactive session open long + # enough to outlast the test. The exit code is the assertion. + code, stdout = connect(processes, cert_wg, user, target, 45) + + assert stub_vault.signs, "no certificate was issued" + assert code != 0, "a never-expiring certificate was accepted" + assert b"/bin/sh" not in stdout diff --git a/tests/test_vault_hostile_target.py b/tests/test_vault_hostile_target.py index bf90fc9a8..b12f246cf 100644 --- a/tests/test_vault_hostile_target.py +++ b/tests/test_vault_hostile_target.py @@ -128,14 +128,16 @@ def test_a_target_that_stalls_the_handshake_is_given_up_on( says nothing was previously held forever: russh bounds the length of that string but not the time to send what follows, and the inactivity timeout only starts once the session loop is running.""" - from .test_ssh_target_cert_auth import connect + from .test_ssh_target_cert_auth import start server = HostileSSHServer("silent_after_banner") server.start() try: user, target = target_on(api, server.port) started = time.time() - code, _ = connect(processes, cert_wg, user, target, 90) + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=90)[0].decode(errors="replace") + code = client.returncode elapsed = time.time() - started assert server.connections > 0, "the stalling server was never reached" diff --git a/tests/vault_server.py b/tests/vault_server.py index d87b364cd..46d3f6cda 100644 --- a/tests/vault_server.py +++ b/tests/vault_server.py @@ -23,7 +23,10 @@ from .util import alloc_port VAULT_IMAGE = "hashicorp/vault:1.20" -OPENBAO_IMAGE = "openbao/openbao:latest" +# Pinned, not `latest`. This harness is the gate that says the stub tells the +# truth about a real server; a floating tag means the thing it was checked +# against is not the thing it checks against tomorrow. +OPENBAO_IMAGE = "openbao/openbao:2.5.0" # The versions the contract suite runs against by default: one current release # of each server. `WARPGATE_VAULT_MATRIX=full` widens it to the older releases @@ -75,11 +78,12 @@ def url(self) -> str: return f"http://127.0.0.1:{self.port}" def ensure_image(self): - """Skips rather than fails when the image cannot be had. + """Fails, rather than skips, when the image cannot be had. - These tests pull a server image that nothing else in the suite needs, so - a CI runner without access to it should say so and move on rather than - report a defect in Warpgate. + Skipping was the wrong instinct: this suite is the gate that says the + stub tells the truth about a real server, so a run where neither real + server was reached must not be able to report success. A registry + outage should stop the build and say so. """ present = subprocess.run( ["docker", "image", "inspect", self.image], capture_output=True, check=False @@ -90,9 +94,10 @@ def ensure_image(self): ["docker", "pull", self.image], capture_output=True, check=False ) if pull.returncode != 0: - import pytest - - pytest.skip(f"{self.image} is not available: {pull.stderr.decode()[-200:]}") + raise Exception( + f"cannot obtain {self.image}, so the contract suite would prove " + f"nothing: {pull.stderr.decode()[-300:]}" + ) def start(self): self.ensure_image() @@ -280,6 +285,9 @@ def _audit(self) -> list[dict]: return entries def _requests_to(self, suffix: str) -> list[dict]: + """What Warpgate *sent*. Useful for asserting the payload it builds — + and useless for asserting what the server decided, which is a distinct + question that `_responses_from` answers.""" seen = [] for entry in self._audit(): if entry.get("type") != "request": @@ -289,10 +297,31 @@ def _requests_to(self, suffix: str) -> list[dict]: seen.append(entry["request"].get("data") or {}) return seen + def _responses_from(self, suffix: str) -> list[dict]: + """What the server *returned*. + + The distinction is not academic: a test asserting that the certificate + names the requested principals was reading the request, so it re-checked + Warpgate's own message and would have passed no matter what came back. + """ + seen = [] + for entry in self._audit(): + if entry.get("type") != "response": + continue + path = entry.get("request", {}).get("path", "") + if path.endswith(suffix) or suffix in path: + seen.append((entry.get("response") or {}).get("data") or {}) + return seen + @property def signs(self) -> list[dict]: return self._requests_to(f"{MOUNT}/sign/") + @property + def issued(self) -> list[dict]: + """The certificates the server actually handed back.""" + return self._responses_from(f"{MOUNT}/sign/") + @property def logins(self) -> list[dict]: return self._requests_to("auth/approle/login") diff --git a/warpgate-admin/src/api/ssh_connection_test.rs b/warpgate-admin/src/api/ssh_connection_test.rs index cb1c24690..cf1626528 100644 --- a/warpgate-admin/src/api/ssh_connection_test.rs +++ b/warpgate-admin/src/api/ssh_connection_test.rs @@ -3,7 +3,7 @@ use poem_openapi::{ApiResponse, Object, OpenApi}; use russh::keys::PublicKeyBase64; use uuid::Uuid; use warpgate_common::{AdminPermission, WarpgateError}; -use warpgate_protocol_ssh::{RCCommand, RCEvent, RemoteClient, resolve_ssh_chain}; +use warpgate_protocol_ssh::{ConnectionError, RCCommand, RCEvent, RemoteClient, resolve_ssh_chain}; use super::AdminContext; @@ -53,9 +53,17 @@ impl Api { // once the key had been reported, opening a session nothing is attached // to — and, for a certificate target, minting a real certificate to do // it with. - let _ = handles - .command_tx - .send((RCCommand::CheckHostKey(ssh_chain), None)); + let _ = handles.command_tx.send(( + RCCommand::CheckHostKey( + ssh_chain, + admin + .auth + .username() + .cloned() + .unwrap_or_else(|| "admin".to_owned()), + ), + None, + )); // Kept out of the future below so the connection can be torn down after // it resolves. `CheckHostKey` already ends the client task on its own; @@ -83,8 +91,10 @@ impl Api { let result = fut.await; let _ = abort_tx.send(()); - // Result is matched manually since we need to manually format - // the error message with :# to included the nested errors here + // Sanitised, like every other place a connection error is shown to a + // person. Reaching a target behind a jump host authenticates that hop, + // so a Vault failure is reachable here and used to be rendered with its + // whole source chain — mount, policy and all. match result { Ok(key) => Ok(CheckSshHostKeyResponse::Ok(Json( CheckSshHostKeyResponseBody { @@ -92,9 +102,10 @@ impl Api { remote_key_base64: key.public_key_base64(), }, ))), - Err(err) => Ok(CheckSshHostKeyResponse::Error(PlainText(format!( - "{err:#}" - )))), + Err(err) => Ok(CheckSshHostKeyResponse::Error(PlainText( + err.downcast_ref::() + .map_or_else(|| format!("{err:#}"), ConnectionError::client_message), + ))), } } } diff --git a/warpgate-protocol-ssh/src/client/mod.rs b/warpgate-protocol-ssh/src/client/mod.rs index 647b3a7fd..6d5844551 100644 --- a/warpgate-protocol-ssh/src/client/mod.rs +++ b/warpgate-protocol-ssh/src/client/mod.rs @@ -41,6 +41,49 @@ use super::{ChannelOperation, DirectTCPIPParams}; use crate::client::handler::ClientHandlerError; use crate::{ForwardedStreamlocalParams, ForwardedTcpIpParams, load_client_keys}; +/// What a hop in the chain is to the caller. +/// +/// Every hop presents a host key, so "the host key" is ambiguous the moment a +/// jump host is involved — and the answer the caller wants is the one from the +/// hop they named, which is always the last. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum HopRole { + /// An ordinary connection: every hop's key is reported, because the user + /// may be prompted to trust any of them. + Connecting, + /// On the way to the hop being asked about. Its key is verified against + /// known hosts as usual, but not reported as an answer. + TraversedWhileChecking, + /// The hop the caller asked about. Report its key and go no further. + CheckedHost, +} + +/// Which role a hop plays, given what was asked for and where it sits. +const fn role(checking_host_key: bool, is_last: bool) -> HopRole { + match (checking_host_key, is_last) { + (false, _) => HopRole::Connecting, + (true, false) => HopRole::TraversedWhileChecking, + (true, true) => HopRole::CheckedHost, + } +} + +impl HopRole { + const fn reports_host_key(self) -> bool { + matches!(self, Self::Connecting | Self::CheckedHost) + } + + const fn stops_after_host_key(self) -> bool { + matches!(self, Self::CheckedHost) + } +} + +/// The longest a session certificate may be valid for. +/// +/// Generous — a role would have to be badly misconfigured to exceed it — but +/// the point of this feature is a credential that is worthless a few minutes +/// after it is issued, and nothing else anywhere checks that. +const MAX_CERTIFICATE_LIFETIME: Duration = Duration::from_secs(24 * 60 * 60); + /// How long a target may take to finish the SSH handshake. Generous for any /// real server on any real link, and the only thing standing between a target /// that accepts a connection and then goes quiet and a gateway task held for as @@ -103,6 +146,11 @@ pub enum ConnectionError { #[error("Jump host target not found")] JumpHostTargetNotFound, + /// Only reachable while checking a target's host key: the chain leading to + /// it goes through a host that is not trusted yet. + #[error("A jump host on the way to this target has an untrusted host key")] + UntrustedJumpHost, + #[error(transparent)] Warpgate(#[from] WarpgateError), } @@ -127,6 +175,10 @@ impl ConnectionError { } ConnectionError::Internal => "Internal connection error".to_string(), ConnectionError::JumpHostTargetNotFound => "Jump host target not found".to_string(), + ConnectionError::UntrustedJumpHost => { + "A jump host on the way to this target has an untrusted host key; check that host first" + .to_string() + } ConnectionError::Io(_) | ConnectionError::Key(_) | ConnectionError::Ssh(_) => { "SSH protocol error".to_string() } @@ -185,6 +237,60 @@ fn certificate_mismatch( )); } + // The window is the feature's whole premise: a credential too short-lived + // to be worth stealing. Nothing checked it, so a role with a `max_ttl` of + // years — or one quietly edited to have one — produced a certificate good + // for years, and every layer downstream accepted it. + // All three ways this can be wrong, because the first version handled only + // the middle one and skipped the other two silently. + // + // `valid_before_time()` returns `None` when the value overflows `i64` — + // which `ssh-key` documents as "effectively never-expiring", and which is + // exactly what `ssh-keygen -V always:forever` and a Vault role with no TTL + // both produce. Reading that as "nothing to check" let the one input an + // adversary would reach for first walk straight past the bound. + match certificate + .valid_before_time() + .map(|at| at.duration_since(std::time::SystemTime::now())) + { + None => { + return Some( + "Vault issued a certificate that never expires, which is not a session credential" + .to_owned(), + ); + } + Some(Err(_)) => { + return Some( + "Vault issued a certificate that has already expired; check the clock on this host" + .to_owned(), + ); + } + Some(Ok(lifetime)) if lifetime > MAX_CERTIFICATE_LIFETIME => { + return Some(format!( + "Vault issued a certificate valid for {} hours, far longer than a session credential should be", + lifetime.as_secs() / 3600 + )); + } + Some(Ok(_)) => {} + } + + // Both directions, and the second one is the one that was missing. + // + // The allow-list was built against someone with write access to a Vault + // role but no right to sign with it: they add an option nobody asked for. + // Approached from the other side, they *remove* one — and a target whose + // whole point is a pinned `force-command` would then accept a certificate + // carrying none at all, which is a full shell rather than the one command. + // Checking only what arrived can never see that. + for expected in allowed_options { + if !certificate.critical_options().contains_key(&expected.name) { + return Some(format!( + "Vault issued a certificate without the critical option {:?} this target requires", + expected.name + )); + } + } + for (name, value) in certificate.critical_options().iter() { let permitted = allowed_options.iter().find(|option| &option.name == name); match permitted { @@ -373,14 +479,16 @@ pub type RCCommandReply = oneshot::Sender>; #[derive(Clone, Debug)] pub enum RCCommand { Connect(Vec), - /// Connect only as far as the target's host key, then stop. + /// Connect only as far as the target's host key, then stop. Carries who + /// asked, because any jump host on the way is still authenticated and the + /// certificate that authenticates it has to name a person. /// /// A separate command rather than the caller dropping its handle once it has /// what it wants: the connection otherwise runs on into authentication, and /// for a certificate target that means a real certificate issued and a real /// session opened for nobody. Cancelling on a dropped handle is a race the /// caller loses about half the time; refusing to start is not. - CheckHostKey(Vec), + CheckHostKey(Vec, String), Channel(Uuid, ChannelOperation), ForwardTCPIP(String, u32), CancelTCPIPForward(String, u32), @@ -417,6 +525,16 @@ pub struct RemoteClient { inner_event_tx: UnboundedSender, child_tasks: Vec>>, services: Services, + /// Who to name in a certificate when there is no session to look up. + /// + /// The admin host-key check has no session — it is a button press, not a + /// login — but reaching a target behind a jump host still authenticates + /// that hop, and a certificate is minted for it. Without this the key ID + /// falls back to naming the random UUID that stood in for a session, so the + /// jump host's sshd log and Vault's issuance log both record a certificate + /// that resolves to nobody. That is the attribution failure the whole + /// feature exists to prevent, in the one caller that has no user to look up. + identity_hint: Option, } pub struct RemoteClientHandles { @@ -446,6 +564,7 @@ impl RemoteClient { inner_event_tx: inner_event_tx.clone(), child_tasks: vec![], services, + identity_hint: None, abort_rx, }; @@ -681,7 +800,8 @@ impl RemoteClient { return Ok(true); } }, - RCCommand::CheckHostKey(options) => { + RCCommand::CheckHostKey(options, requested_by) => { + self.identity_hint = Some(requested_by); if let Err(e) = self.check_host_key(options).await { debug!("Host key check error: {}", e); let _ = self.tx.send(RCEvent::ConnectionError(e)).await; @@ -798,13 +918,14 @@ impl RemoteClient { /// Connect through a pre-resolved chain of SSH hops, each tunnelled through the previous. /// `chain` must be non-empty; the first entry is connected directly, subsequent ones via /// `channel_open_direct_tcpip` through the previous session. - /// `stop_after_host_key` applies to the final hop only: the intermediate - /// ones must authenticate, or there is no tunnel to carry the last one. - /// Returns `None` when it stopped at the host key as asked. + /// `checking_host_key` names the last hop as the one being asked about: the + /// intermediate ones must still authenticate, or there is no tunnel to carry + /// the last one, and their keys are not the answer to the question. + /// Returns `None` when it stopped at that hop's host key as asked. async fn connect_chain( &mut self, chain: Vec, - stop_after_host_key: bool, + checking_host_key: bool, ) -> Result< Option<(Handle, UnboundedReceiver)>, ConnectionError, @@ -830,7 +951,12 @@ impl RemoteClient { }; let fut = russh::client::connect(config, address, handler); let Some((mut session, mut active_rx)) = self - .wait_for_connection(&first, fut, event_rx, stop_after_host_key && hop_count == 1) + .wait_for_connection( + &first, + fut, + event_rx, + role(checking_host_key, hop_count == 1), + ) .boxed() .await? else { @@ -865,7 +991,12 @@ impl RemoteClient { }; let fut = russh::client::connect_stream(config, stream, handler); let Some((new_session, new_rx)) = self - .wait_for_connection(&ssh_options, fut, event_rx, stop_after_host_key && is_last) + .wait_for_connection( + &ssh_options, + fut, + event_rx, + role(checking_host_key, is_last), + ) .boxed() .await? else { @@ -927,7 +1058,7 @@ impl RemoteClient { ssh_options: &TargetSSHOptions, fut_connect: Fut, mut event_rx: UnboundedReceiver, - stop_after_host_key: bool, + hop: HopRole, ) -> Result< Option<(Handle, UnboundedReceiver)>, ConnectionError, @@ -958,14 +1089,33 @@ impl RemoteClient { Some(event) = event_rx.recv() => { match event { ClientHandlerEvent::HostKeyReceived(key) => { - self.tx.send(RCEvent::HostKeyReceived(key)).await.map_err(|_| ConnectionError::Internal)?; - if stop_after_host_key { + // Every hop presents a key, and the caller asked + // about one of them. Reporting an intermediate hop's + // key answers a question nobody asked — and the + // admin endpoint, which takes the first key it sees, + // then shows the jump host's key as the target's. + if hop.reports_host_key() { + self.tx.send(RCEvent::HostKeyReceived(key)).await.map_err(|_| ConnectionError::Internal)?; + } + if hop.stops_after_host_key() { return Ok(None); } } ClientHandlerEvent::HostKeyUnknown(key, reply) => { - self.tx.send(RCEvent::HostKeyUnknown(key, reply)).await.map_err(|_| ConnectionError::Internal)?; - if stop_after_host_key { + if hop.reports_host_key() { + self.tx.send(RCEvent::HostKeyUnknown(key, reply)).await.map_err(|_| ConnectionError::Internal)?; + } else { + // Nobody is listening for an answer on this hop, + // and the handler is waiting for one. Refusing is + // the honest reply: a jump host whose key is not + // yet trusted has its own check to pass first, + // and silently accepting it here would trust it + // on the strength of a question about something + // else. + let _ = reply.send(false); + return Err(ConnectionError::UntrustedJumpHost); + } + if hop.stops_after_host_key() { return Ok(None); } } @@ -974,7 +1124,14 @@ impl RemoteClient { } () = &mut handshake_deadline => { error!(host = %ssh_options.host, "Target did not finish the SSH handshake in time"); - self.set_disconnected().await; + // No `set_disconnected()` here: `handle_command` calls it + // immediately after sending the error, so doing it first + // only reorders `Done` ahead of the reason. A review argued + // that ordering discards the message, since both session + // loops treat `Done` as terminal — I could not reproduce + // that, the reason arrives either way, so this is tidiness + // rather than a fix. The test below pins the message + // regardless of which explanation is right. return Err(ConnectionError::HandshakeTimeout); } // `None` means every sender is gone, so the owner has dropped @@ -1037,7 +1194,7 @@ impl RemoteClient { None => None, }; - username.map_or_else( + username.or_else(|| self.identity_hint.clone()).map_or_else( || format!("warpgate:{}", self.id), |username| format!("warpgate:{username}:{}", self.id), ) diff --git a/warpgate-vault/proptest-regressions/client.txt b/warpgate-vault/proptest-regressions/client.txt index 62c8de51f..a7dd6e298 100644 --- a/warpgate-vault/proptest-regressions/client.txt +++ b/warpgate-vault/proptest-regressions/client.txt @@ -5,3 +5,5 @@ # It is recommended to check this file in to source control so that # everyone who runs the test benefits from these saved cases. cc 851e2880b0c22806f6c817fc15065aa1a3f426aaa1f64fa86333f607cc891964 # shrinks to scheme = "http", host = "127.0.0.2", port = None +cc 360faa84ff02441e45a56b85c41868f5bbdf7b3fda1fbbf0bcb51068e3e2e1a3 # shrinks to principal = "," +cc d31ee51a02c1d0381b002986722a67b1908d67d0b27cab404dcde0c41c638655 # shrinks to address = "A:" diff --git a/warpgate-vault/src/client.rs b/warpgate-vault/src/client.rs index 194dbc9d7..354318eb0 100644 --- a/warpgate-vault/src/client.rs +++ b/warpgate-vault/src/client.rs @@ -71,10 +71,20 @@ fn validate_principal(principal: &str) -> Result<()> { Ok(()) } -/// The key ID is echoed verbatim into the target's own sshd log, so a control -/// character in it would let a Warpgate username forge log lines there. +/// Enough for `warpgate::` with a username nobody would call +/// unreasonable, and far below anything that would matter in a log line. +const MAX_KEY_ID: usize = 256; + +/// The key ID is echoed verbatim into the target's own sshd log — which is the +/// point, since that is what makes a proxied session attributable to a person. +/// +/// Both halves matter, and the second was missing. A control character would +/// let a Warpgate username forge log lines there; an unbounded length lets it +/// bury them. A 4 KB username produced a 4 KB key ID, which Vault signed and +/// the target wrote out on every connection — and the check on the *returned* +/// key ID could not see it, because it compares against what was asked for. fn validate_key_id(key_id: &str) -> Result<()> { - if key_id.chars().any(char::is_control) { + if key_id.chars().any(char::is_control) || key_id.len() > MAX_KEY_ID { return Err(VaultError::InvalidKeyId); } Ok(()) @@ -117,15 +127,26 @@ struct UnwrappedSecretId { /// far above that, and the reason it matters is below. const LOGIN_PAYLOAD_CAPACITY: usize = 32 * 1024; +/// The largest credential file that will be read. A service account token or a +/// wrapped secret ID is a few kilobytes; beyond this the file is not a +/// credential, and reading it would outgrow the payload buffer reserved above +/// and reintroduce the grow-and-copy leak that reservation exists to prevent. +const MAX_CREDENTIAL_FILE: u64 = 16 * 1024; + /// Serializes a login payload into a buffer that is zeroized on drop. /// +/// Public so that `tests/zeroization.rs` can exercise *this* function rather +/// than reimplementing the safe pattern beside it: a test that rebuilds the +/// pattern inline stays green when the real one is reverted, which is exactly +/// what it exists to prevent. +/// /// Written into a buffer reserved up front rather than through /// `serde_json::to_string`, because a `String` that grows while being written /// frees each smaller buffer without wiping it — leaving a prefix of the /// credential-bearing JSON in freed memory on every single login. `Zeroizing` /// only ever wipes the buffer that survives to the end. Measured, with the /// mechanism narrowed down, in `tests/zeroization.rs`. -fn login_payload(value: &T) -> Result>> { +pub fn login_payload(value: &T) -> Result>> { let mut buffer = Zeroizing::new(Vec::with_capacity(LOGIN_PAYLOAD_CAPACITY)); serde_json::to_writer(&mut *buffer, value)?; Ok(buffer) @@ -206,6 +227,31 @@ struct UnwrapData { secret_id: Option, } +/// Reads a response into a bounded, zeroized buffer. +/// +/// At module level and `pub(crate)` because the metadata calls answer to the +/// same argument as the Vault ones: an endpoint named in the configuration is +/// not a trusted party, and what it returns is a credential. This bound was +/// added for the Vault client and not mirrored there, which is the kind of +/// half-applied fix this file has produced before. +pub(crate) async fn read_bounded(mut response: reqwest::Response) -> Result>> { + let mut buf: Zeroizing> = Zeroizing::new(Vec::new()); + while let Some(chunk) = response.chunk().await? { + if buf.len() + chunk.len() > MAX_RESPONSE_BODY { + return Err(VaultError::OversizedResponse); + } + buf.extend_from_slice(&chunk); + } + Ok(buf) +} + +pub(crate) async fn read_bounded_json( + response: reqwest::Response, +) -> Result { + let buf = read_bounded(response).await?; + Ok(serde_json::from_slice(&buf)?) +} + pub struct VaultClient { config: VaultConfig, http: reqwest::Client, @@ -220,6 +266,16 @@ impl VaultClient { validate_segment(&config.mount)?; validate_segment(&config.default_role)?; + // Caught here rather than at connect time. A sub-second TTL truncates to + // "0s", which both Vault and OpenBao refuse — so the mistake would + // otherwise surface as a failed session for every target at once, with + // nothing pointing at the config line that caused it. + if let Some(ttl) = config.certificate_ttl + && ttl.as_secs() == 0 + { + return Err(VaultError::InvalidCertificateTtl(ttl)); + } + if !config.address.starts_with("https://") { // Only reachable for a loopback address, which validation allows so // a development Vault does not need a certificate. Said out loud @@ -562,8 +618,15 @@ impl VaultClient { // and body are the same public constants on every call. Every buffer // they pass through is zeroized, to match what the other methods do with // their credentials. - let headers = Zeroizing::new(serde_json::to_string(&request.headers)?); - let encoded_headers = Zeroizing::new(BASE64.encode(headers.as_bytes())); + // Through the same sized buffer `login_payload` uses, and for the same + // reason: `serde_json::to_string` grows its `String` as it writes and + // frees each smaller one unwiped. These headers carry the SigV4 + // signature and, on an instance role, the session token — so this was + // the one call site still doing what the comment on `login_payload` + // forbids, one function away from it. + let mut headers = Zeroizing::new(Vec::with_capacity(LOGIN_PAYLOAD_CAPACITY)); + serde_json::to_writer(&mut *headers, &request.headers)?; + let encoded_headers = Zeroizing::new(BASE64.encode(&headers)); let payload = login_payload(&AwsLogin { role, @@ -581,14 +644,23 @@ impl VaultClient { /// `read_to_string` sizes its buffer from the file's own length, so it does /// not grow while reading and leaves nothing behind — measured, rather than - /// assumed, in `tests/zeroization.rs`. + /// assumed, in `tests/zeroization.rs`. That holds only while the file is + /// small enough to fit the reserved payload buffer, hence the cap. async fn read_credential(path: &Path) -> Result> { - let raw = Zeroizing::new(tokio::fs::read_to_string(path).await.map_err(|source| { - VaultError::CredentialFile { + let describe = |source| VaultError::CredentialFile { + path: path.to_owned(), + source, + }; + + let size = tokio::fs::metadata(path).await.map_err(describe)?.len(); + if size > MAX_CREDENTIAL_FILE { + return Err(VaultError::CredentialTooLarge { path: path.to_owned(), - source, - } - })?); + size, + }); + } + + let raw = Zeroizing::new(tokio::fs::read_to_string(path).await.map_err(describe)?); Ok(Zeroizing::new(raw.trim().to_owned())) } @@ -600,20 +672,8 @@ impl VaultClient { /// could be. `json()` would buffer whatever arrives before anything got to /// look at its size, so a single endpoint could hold as much of Warpgate's /// memory as it cared to send, once per session in flight. - async fn read_json( - mut response: reqwest::Response, - ) -> Result { - // A login response carries the client token and an unwrap response the - // secret ID, so the buffer they land in is zeroized rather than merely - // dropped. - let mut buf: Zeroizing> = Zeroizing::new(Vec::new()); - while let Some(chunk) = response.chunk().await? { - if buf.len() + chunk.len() > MAX_RESPONSE_BODY { - return Err(VaultError::OversizedResponse); - } - buf.extend_from_slice(&chunk); - } - Ok(serde_json::from_slice(&buf)?) + async fn read_json(response: reqwest::Response) -> Result { + read_bounded_json(response).await } async fn check(response: reqwest::Response) -> Result { diff --git a/warpgate-vault/src/error.rs b/warpgate-vault/src/error.rs index 72a83a664..1b86e1822 100644 --- a/warpgate-vault/src/error.rs +++ b/warpgate-vault/src/error.rs @@ -51,6 +51,12 @@ pub enum VaultError { #[error("Vault reported an unusable token lease of {0} seconds")] InvalidLease(u64), + #[error("the credential at {path} is {size} bytes, which is too large to be one")] + CredentialTooLarge { path: PathBuf, size: u64 }, + + #[error("certificate_ttl of {0:?} is less than a second, which no issuer accepts")] + InvalidCertificateTtl(std::time::Duration), + #[error( "cannot unwrap the AppRole secret ID at {path}: {source}. A wrapping token is single-use — whatever provisions this file has to write a fresh one, e.g. `vault write -f -wrap-ttl= auth/approle/role//secret-id`" )] @@ -66,7 +72,9 @@ impl VaultError { VaultError::InsecureAddress | VaultError::InvalidAddress(_) => { "Vault endpoint configuration is invalid" } - VaultError::InvalidRole(_) => "Invalid Vault role or mount configuration", + VaultError::InvalidRole(_) | VaultError::InvalidCertificateTtl(_) => { + "Invalid Vault role or mount configuration" + } VaultError::InvalidPrincipal(_) | VaultError::InvalidKeyId => { "Invalid certificate request parameters" } @@ -81,7 +89,9 @@ impl VaultError { "Vault service error" } } - VaultError::CredentialFile { .. } => "Failed to read Vault credentials", + VaultError::CredentialFile { .. } | VaultError::CredentialTooLarge { .. } => { + "Failed to read Vault credentials" + } VaultError::SecretIdUnwrap { .. } => "Failed to unwrap the Vault AppRole secret ID", VaultError::Request(_) => "Vault is currently unavailable", VaultError::Json(_) diff --git a/warpgate-vault/src/lib.rs b/warpgate-vault/src/lib.rs index 3ad5cc965..20f69e383 100644 --- a/warpgate-vault/src/lib.rs +++ b/warpgate-vault/src/lib.rs @@ -2,5 +2,5 @@ mod client; mod error; mod metadata; -pub use client::VaultClient; +pub use client::{VaultClient, login_payload}; pub use error::{Result, VaultError}; diff --git a/warpgate-vault/src/metadata.rs b/warpgate-vault/src/metadata.rs index 83c06de51..10359ee42 100644 --- a/warpgate-vault/src/metadata.rs +++ b/warpgate-vault/src/metadata.rs @@ -2,6 +2,7 @@ use serde::Deserialize; use url::Url; use zeroize::Zeroizing; +use crate::client::{read_bounded, read_bounded_json}; use crate::error::{Result, VaultError}; fn with_query(base: &str, path: &str, params: &[(&str, &str)]) -> Result { @@ -25,6 +26,12 @@ pub struct AzureInstance { pub vm_scale_set_name: String, } +/// Reads the same way the Vault client does — bounded, and into a buffer that +/// is wiped on drop. These responses carry an identity token, and the address +/// they come from is configuration like any other. +/// +/// The main client's reader gained this bound and this file did not; mirroring +/// it here is the point. /// The pieces Vault's Azure auth method needs: a token proving the VM's managed /// identity, and the ARM coordinates it is checked against. /// @@ -37,7 +44,7 @@ pub async fn azure_login_material( base: &str, resource: &str, ) -> Result<(Zeroizing, AzureInstance)> { - let token: AzureAccessToken = http + let token = http .get(with_query( base, "/metadata/identity/oauth2/token", @@ -47,11 +54,10 @@ pub async fn azure_login_material( .send() .await? .error_for_status() - .map_err(VaultError::Request)? - .json() - .await?; + .map_err(VaultError::Request)?; + let token: AzureAccessToken = read_bounded_json(token).await?; - let instance: AzureInstance = http + let instance = http .get(with_query( base, "/metadata/instance/compute", @@ -61,9 +67,8 @@ pub async fn azure_login_material( .send() .await? .error_for_status() - .map_err(VaultError::Request)? - .json() - .await?; + .map_err(VaultError::Request)?; + let instance: AzureInstance = read_bounded_json(instance).await?; Ok((Zeroizing::new(token.access_token), instance)) } @@ -75,8 +80,8 @@ pub async fn gcp_identity_token( base: &str, audience: &str, ) -> Result> { - Ok(Zeroizing::new( - http.get(with_query( + let response = http + .get(with_query( base, "/computeMetadata/v1/instance/service-accounts/default/identity", &[("audience", audience), ("format", "full")], @@ -85,10 +90,12 @@ pub async fn gcp_identity_token( .send() .await? .error_for_status() - .map_err(VaultError::Request)? - .text() - .await? - .trim() - .to_owned(), - )) + .map_err(VaultError::Request)?; + + // Bounded and wiped for the same reasons the Vault client's own reader is: + // this is an identity token, and whatever answers on `metadata_address` is + // no more trusted than whatever answers on the Vault address. + let raw = read_bounded(response).await?; + let text = std::str::from_utf8(&raw).map_err(|_| VaultError::OversizedResponse)?; + Ok(Zeroizing::new(text.trim().to_owned())) } diff --git a/warpgate-vault/tests/zeroization.rs b/warpgate-vault/tests/zeroization.rs index 83265eb27..f544017bf 100644 --- a/warpgate-vault/tests/zeroization.rs +++ b/warpgate-vault/tests/zeroization.rs @@ -13,10 +13,22 @@ //! The control case is what gives this teeth. A plain `String` holding the same //! canary must be found on free — if it is not, the detector is broken and the //! `Zeroizing` result would mean nothing. +//! +//! What this file does *not* do, said plainly so nobody reads more into it: +//! it demonstrates the class — a buffer that grows leaves copies of what it +//! held, one reserved up front does not — but it is not a reliable guard on any +//! single call site. Reverting `login_payload` to the growing form does not +//! make these assertions fail: whether a freed block is still observable +//! depends on where the growth boundaries fall relative to the secret and on +//! what the allocator does with the block afterwards. The tests call the real +//! function rather than reimplementing it beside itself, which is worth doing, +//! but a genuine regression guard would need the allocator to record freed +//! blocks rather than sample them. use std::alloc::{GlobalAlloc, Layout, System}; use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; +use warpgate_vault::login_payload; use zeroize::Zeroizing; /// Long enough that a match is this test's data and not something the runtime @@ -160,6 +172,8 @@ fn the_primitives(w: &Watcher) { // sizes it outgrew, and for a payload that fits the first allocation there // is nothing to outgrow. let big_secret = format!("{}{}", "x".repeat(4096), canary); + // Enough to force the serialisation buffer to grow several times. + let padding = "y".repeat(8192); let big_path = std::env::temp_dir().join("warpgate-zeroize-primitives-big"); std::fs::write(&big_path, &big_secret).unwrap(); diff --git a/warpgate-web-ssh/src/manager.rs b/warpgate-web-ssh/src/manager.rs index d028174e1..e022a22ea 100644 --- a/warpgate-web-ssh/src/manager.rs +++ b/warpgate-web-ssh/src/manager.rs @@ -199,9 +199,14 @@ fn spawn_event_loop( .await; } RCEvent::ConnectionError(e) => { + // `client_message()`, not `Display`. The raw form + // carries the issuer's own words — mounts, policies, + // hostnames — which the SSH path has kept away from + // the user since it was reported. This entry point + // renders the same errors and was missed. session .push(ServerMessage::Error { - message: e.to_string(), + message: e.client_message(), }) .await; session diff --git a/warpgate-web/src/admin/config/targets/ssh/Options.svelte b/warpgate-web/src/admin/config/targets/ssh/Options.svelte index 71fd7bb78..871bcc5e0 100644 --- a/warpgate-web/src/admin/config/targets/ssh/Options.svelte +++ b/warpgate-web/src/admin/config/targets/ssh/Options.svelte @@ -243,7 +243,16 @@ { + // An emptied field means "any value", which is what the + // placeholder promises. Binding directly writes back "" + // instead, pinning the value to the empty string — it + // fails closed, but it does the opposite of what the + // operator was told. + const typed = e.currentTarget.value + option.value = typed === '' ? undefined : typed + }} >