diff --git a/.gitignore b/.gitignore index b2d571632..962be5f99 100644 --- a/.gitignore +++ b/.gitignore @@ -36,3 +36,6 @@ node_modules .github *.md .env + +# rustc crash dumps land in the working directory and are not ours to keep +rustc-ice-*.txt diff --git a/Cargo.lock b/Cargo.lock index 1fa4ecb7b..03fbbd3b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1067,12 +1067,27 @@ dependencies = [ "bit-vec 0.6.3", ] +[[package]] +name = "bit-set" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08807e080ed7f9d5433fa9b275196cfc35414f66a0c79d864dc51a0d825231a3" +dependencies = [ + "bit-vec 0.8.0", +] + [[package]] name = "bit-vec" version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "349f9b6a179ed607305526ca489b34ad0a41aed5f7980fa90eb03160b69598fb" +[[package]] +name = "bit-vec" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" + [[package]] name = "bit-vec" version = "0.9.1" @@ -2429,7 +2444,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b95f7c0680e4142284cf8b22c14a476e87d61b004a3a0861872b32ef7ead40a2" dependencies = [ - "bit-set", + "bit-set 0.5.3", "regex", ] @@ -5827,6 +5842,25 @@ dependencies = [ "yansi", ] +[[package]] +name = "proptest" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b45fcc2344c680f5025fe57779faef368840d0bd1f42f216291f0dc4ace4744" +dependencies = [ + "bit-set 0.8.0", + "bit-vec 0.8.0", + "bitflags 2.13.1", + "num-traits", + "rand 0.9.4", + "rand_chacha 0.9.0", + "rand_xorshift", + "regex-syntax", + "rusty-fork", + "tempfile", + "unarray", +] + [[package]] name = "prost" version = "0.14.4" @@ -5883,6 +5917,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "quick-error" +version = "1.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" + [[package]] name = "quick-xml" version = "0.36.2" @@ -6006,6 +6046,15 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rand_xorshift" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a" +dependencies = [ + "rand_core 0.9.5", +] + [[package]] name = "ratatui" version = "0.30.2" @@ -6736,6 +6785,18 @@ version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" +[[package]] +name = "rusty-fork" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc6bf79ff24e648f6da1f8d1f011e9cac26491b619e6b9280f2b47f1774e6ee2" +dependencies = [ + "fnv", + "quick-error", + "tempfile", + "wait-timeout", +] + [[package]] name = "ryu" version = "1.0.23" @@ -8636,6 +8697,12 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971" +[[package]] +name = "unarray" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94" + [[package]] name = "uncased" version = "0.9.10" @@ -8870,6 +8937,15 @@ dependencies = [ "utf8parse", ] +[[package]] +name = "wait-timeout" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ac3b126d3914f9849036f826e054cbabdc8519970b8998ddaf3b5bd3c65f11" +dependencies = [ + "libc", +] + [[package]] name = "walkdir" version = "2.5.0" @@ -8939,6 +9015,7 @@ dependencies = [ "warpgate-protocol-ssh", "warpgate-protocol-vnc", "warpgate-tls", + "warpgate-vault", ] [[package]] @@ -9151,6 +9228,7 @@ dependencies = [ "warpgate-ldap", "warpgate-sso", "warpgate-tls", + "warpgate-vault", "webpki", "zune-jpeg", ] @@ -9429,6 +9507,7 @@ dependencies = [ "dialoguer", "ed25519-dalek 3.0.0", "futures", + "humantime", "natord", "ratatui", "russh", @@ -9450,6 +9529,7 @@ dependencies = [ "warpgate-core", "warpgate-db-entities", "warpgate-tls", + "warpgate-vault", "zeroize", ] @@ -9523,6 +9603,29 @@ dependencies = [ "x509-parser", ] +[[package]] +name = "warpgate-vault" +version = "0.27.4" +dependencies = [ + "data-encoding", + "proptest", + "rcgen", + "reqwest 0.13.4", + "rustls 0.23.43", + "rustls-pki-types", + "serde", + "serde_json", + "tempfile", + "thiserror 2.0.20", + "tokio", + "tokio-rustls 0.26.4", + "tracing", + "url", + "warpgate-aws", + "warpgate-common", + "zeroize", +] + [[package]] name = "warpgate-web" version = "0.28.2" diff --git a/Cargo.toml b/Cargo.toml index 6d2503045..87f16034a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -23,6 +23,7 @@ members = [ "warpgate-protocol-vnc", "warpgate-desktop-ui", "warpgate-sso", + "warpgate-vault", "warpgate-web", "warpgate-web-clients-common", "warpgate-web-desktop", diff --git a/config-schema.json b/config-schema.json index fbe685af4..5d969169b 100644 --- a/config-schema.json +++ b/config-schema.json @@ -119,6 +119,17 @@ "$ref": "#/$defs/SsoProviderConfig" } }, + "vault": { + "description": "Absent unless the deployment issues target credentials from Vault.", + "anyOf": [ + { + "$ref": "#/$defs/VaultConfig" + }, + { + "type": "null" + } + ] + }, "vnc": { "$ref": "#/$defs/VncConfig", "default": { @@ -804,6 +815,185 @@ "host_header" ] }, + "VaultAuth": { + "description": "How Warpgate proves its own identity to Vault.\n\nBoth methods read their credential from a file rather than from the config,\ndeliberately: the point of issuing target credentials on demand is that\nnothing long-lived sits on the Warpgate host, and a value pasted into the\nconfig would put it straight back. A Kubernetes service account token is\nmounted and rotated by the kubelet; an AppRole secret ID is short-lived and\nmeant to be delivered response-wrapped by whatever provisions the host.", + "oneOf": [ + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "kubernetes" + }, + "role": { + "type": "string" + }, + "token_path": { + "type": "string", + "default": "/var/run/secrets/kubernetes.io/serviceaccount/token" + } + }, + "required": [ + "kind", + "role" + ] + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "app_role" + }, + "role_id": { + "type": "string" + }, + "secret_id_path": { + "type": "string" + } + }, + "required": [ + "kind", + "role_id", + "secret_id_path" + ] + }, + { + "description": "Signs an `sts:GetCallerIdentity` call with the default credential chain —\non EC2 that is the instance role. Access keys in the environment work but\nput back the long-lived secret this exists to avoid.", + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "aws" + }, + "region": { + "description": "Signs against a regional STS endpoint instead of the global one. Set\nthis only when Vault is configured with a matching `sts_endpoint`:\nVault replays the request globally by default, and a region-scoped\nsignature is rejected there.", + "type": [ + "string", + "null" + ], + "default": null + }, + "role": { + "type": [ + "string", + "null" + ], + "default": null + }, + "server_id": { + "description": "Bound into the signature so a captured request cannot be replayed\nagainst a different Vault. Must match the server's `iam_server_id_header_value`.", + "type": [ + "string", + "null" + ], + "default": null + } + }, + "required": [ + "kind" + ] + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "azure" + }, + "metadata_address": { + "type": "string", + "default": "http://169.254.169.254" + }, + "resource": { + "type": "string", + "default": "https://management.azure.com/" + }, + "role": { + "type": "string" + } + }, + "required": [ + "kind", + "role" + ] + }, + { + "type": "object", + "properties": { + "kind": { + "type": "string", + "const": "gcp" + }, + "metadata_address": { + "type": "string", + "default": "http://metadata.google.internal" + }, + "role": { + "type": "string" + } + }, + "required": [ + "kind", + "role" + ] + } + ] + }, + "VaultConfig": { + "type": "object", + "properties": { + "address": { + "description": "Base URL of the Vault server, e.g. `https://vault.internal:8200`.", + "type": "string" + }, + "auth": { + "$ref": "#/$defs/VaultAuth" + }, + "ca_bundle": { + "description": "PEM file holding the CA that issued Vault's certificate, for a Vault\nbehind a private CA.\n\nAdded to the host's trust store rather than replacing it, so a\nmisconfigured path cannot silently turn verification off — an\nunreadable or malformed file is a startup error. There is deliberately\nno switch to skip verification: the Vault token crosses this connection\nin a header, and unlike the HTTP and Kubernetes target paths, which\noffer `verify: false` for devices whose certificates cannot be fixed,\nthere is no equivalent case here.", + "type": [ + "string", + "null" + ], + "default": null + }, + "ca_public_key": { + "description": "The signing CA the target trusts, in OpenSSH public-key format\n(`ssh-ed25519 AAAA…`), pinned so a certificate signed by anything else\nis refused before it is offered.\n\nEvery other check on Vault's response asks whether the certificate is\nwhat was requested. This is the only one that asks *who signed it*. The\ntarget's `TrustedUserCAKeys` is the real enforcement and would refuse\nsuch a certificate anyway — but it does so after Warpgate has offered\nit, and the refusal that comes back names the target rather than the\nissuer that mis-signed. Pinning turns a confusing rejection into a\nprecise one, and detects a role rebound to a different CA.\n\nLeft unset, nothing is checked here.", + "type": [ + "string", + "null" + ], + "default": null + }, + "certificate_ttl": { + "description": "Lifetime asked for when signing. Vault clamps this to the role's\n`max_ttl`, so it can only shorten what the role already allows — set it\nto hold the window down without editing Vault. Left unset, the role's own\nTTL decides.", + "type": [ + "string", + "null" + ], + "default": null + }, + "default_role": { + "description": "Signing role used by targets that don't name one of their own.", + "type": "string" + }, + "mount": { + "description": "Mount point of the SSH secrets engine in signed-certificates mode.", + "type": "string", + "default": "ssh-client-signer" + }, + "timeout": { + "type": "string", + "default": "10s" + } + }, + "required": [ + "address", + "default_role", + "auth" + ] + }, "VncConfig": { "type": "object", "properties": { diff --git a/tests/conftest.py b/tests/conftest.py index d0f3b657c..a4ace6b30 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -26,6 +26,24 @@ from deepmerge import always_merger from .util import _wait_timeout, alloc_port, wait_port + +# Anchored on this file rather than on the working directory. The rest of the +# suite reaches its keys as `ssh-keys/...` and gets away with it because pytest +# runs from `tests/`; a path handed to `docker -v` does not get away with it, +# because Docker resolves it against a different root and fails quietly. +SUITE_SSH_KEYS = Path(__file__).parent / "ssh-keys" + +# The address a target is configured with, and therefore the one Warpgate dials +# outbound. Not `localhost`: Warpgate resolves a hostname and connects to the +# first address it gets, which on a dual-stack host is `::1`, while the Docker +# containers these tests start publish on v4 only. The connection is then +# refused, the test fails for a reason that has nothing to do with what it +# asserts, and — worse — a guard-disabled run fails identically, which reads as +# the guard being caught. The Python fake servers in this suite were made +# dual-stack for the same underlying reason; that fixed the instances we owned +# and not the class, because nothing can make a published Docker port answer on +# `::1`. +TARGET_HOST = "127.0.0.1" from .test_http_common import echo_server_port # noqa @@ -85,6 +103,15 @@ class Child: RDP_BACKEND_SIZE = (1280, 800) +@dataclass +class SshHostKey: + """The key an `sshd` fixture was started with, in the two fields the admin + API names it by.""" + + key_type: str + base64: str + + @dataclass class WarpgateProcess: config_path: Path @@ -106,6 +133,9 @@ def __init__(self, ctx: Context, timeout: int) -> None: self.ctx = ctx self.timeout = timeout self._k3s_containers: List[str] = [] + # Port to the host key that server was started with, for the servers + # that were given one of their own. + self.host_keys: dict[int, SshHostKey] = {} def _remove_k3s_containers(self): """Force-remove every k3s container we've started so far. Idempotent — @@ -144,18 +174,56 @@ def stop(self): pass p.kill() - def start_ssh_server(self, trusted_keys=[], extra_config=""): + def start_ssh_server( + self, trusted_keys=[], extra_config="", trusted_ca=[], distinct_host_key=False + ): port = alloc_port() data_dir = self.ctx.tmpdir / f"sshd-{uuid.uuid4()}" data_dir.mkdir(parents=True) authorized_keys_path = data_dir / "authorized_keys" authorized_keys_path.write_text("\n".join(trusted_keys)) + # Always written, even when empty: an unreadable TrustedUserCAKeys file + # makes sshd reject every connection, which is indistinguishable from the + # certificate being wrong. + trusted_ca_path = data_dir / "trusted_ca" + trusted_ca_path.write_text("\n".join(trusted_ca)) + # Every server otherwise shares one host key, which makes two of them + # indistinguishable to anything that identifies a host by its key — and + # therefore makes it impossible to tell which hop of a chain answered. + # + # Either way the key ends up inside `data_dir`, which is already bind + # mounted. The shared one used to be reached by mounting the source tree + # itself, `-v {os.getcwd()}/ssh-keys:/ssh-keys` — a host path assembled + # from wherever pytest happened to start. A Docker setup that + # allow-lists which host directories may be shared mounts that as empty + # rather than refusing, so sshd came up with no host key and every + # target container died before the test it was started for could say + # why. Eleven guards were reported as not discriminating for this and + # one other cause, none of them application defects. + own_key = data_dir / "host_key" + if distinct_host_key: + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-f", str(own_key), "-N", ""], + check=True, + ) + # Recorded so a test can assert which host answered, rather than + # only that it was not some other one. With two hops those are the + # same claim; with three they are not, and the weaker one is what + # the host-key tests were making. + key_type, key_base64 = own_key.with_suffix(".pub").read_text().split()[:2] + self.host_keys[port] = SshHostKey(key_type=key_type, base64=key_base64) + else: + shutil.copy(SUITE_SSH_KEYS / "id_ed25519", own_key) + own_key.chmod(0o600) + host_key_path = str(own_key) + config_path = data_dir / "sshd_config" config_path.write_text( dedent( f"""\ Port 22 AuthorizedKeysFile {authorized_keys_path} + TrustedUserCAKeys {trusted_ca_path} AllowAgentForwarding yes AllowTcpForwarding yes GatewayPorts yes @@ -164,7 +232,7 @@ def start_ssh_server(self, trusted_keys=[], extra_config=""): PermitTunnel yes StrictModes no PermitRootLogin yes - HostKey /ssh-keys/id_ed25519 + HostKey {host_key_path} Subsystem sftp /usr/lib/ssh/sftp-server LogLevel DEBUG3 {extra_config} @@ -173,6 +241,7 @@ def start_ssh_server(self, trusted_keys=[], extra_config=""): ) data_dir.chmod(0o700) authorized_keys_path.chmod(0o600) + trusted_ca_path.chmod(0o600) config_path.chmod(0o600) self.start( @@ -182,10 +251,14 @@ def start_ssh_server(self, trusted_keys=[], extra_config=""): "--rm", "-p", f"{port}:22", + # A jump host has to dial the next hop, which is another + # container published on a host port. Docker Desktop invents + # this name on its own, so a chain test passes on a laptop and + # fails on Linux, where nothing defines it. + "--add-host", + "host.docker.internal:host-gateway", "-v", f"{data_dir}:{data_dir}", - "-v", - f"{os.getcwd()}/ssh-keys:/ssh-keys", "warpgate-e2e-ssh-server", "-f", str(config_path), diff --git a/tests/hostile_ssh_server.py b/tests/hostile_ssh_server.py new file mode 100644 index 000000000..1d250dd3f --- /dev/null +++ b/tests/hostile_ssh_server.py @@ -0,0 +1,126 @@ +"""An SSH server that does not intend to be one. + +Every other test in this suite treats the target as honest: a stock `sshd` in a +container that answers correctly or refuses politely. That is the one trust +boundary nothing here has ever pushed on, and it is not a hypothetical — five of +russh's fourteen published advisories are reachable from the peer, including two +pre-authentication panics in the *client* role, which is the role Warpgate plays +here. + +A target is added by an administrator, but it lives on a machine Warpgate does +not own. If a compromised host can hang or crash the gateway's client, it takes +down more than its own session — and the certificate feature has Warpgate dial +out, with a freshly minted credential, on every connection. + +Each mode below is a way of being wrong that a real server never is. +""" + +import socket +import threading + +from .util import alloc_port + +MODES = { + # RFC 4253 says the identification string ends with CR LF. Without it a + # client that reads "until the line ends" reads forever. + "banner_never_ends": None, + # Bounded but enormous. A client that buffers the banner before validating + # its length allocates all of it. + "banner_gigantic": None, + # Correct banner, then bytes that are not a packet. + "garbage_after_banner": None, + # Correct banner, then nothing at all, forever. + "silent_after_banner": None, + # Accept and close immediately. + "instant_close": None, + # A packet claiming a length far beyond anything real. + "absurd_packet_length": None, +} + + +class HostileSSHServer: + """Listens on a port and misbehaves in one chosen way.""" + + def __init__(self, mode: str): + if mode not in MODES: + raise ValueError(f"unknown mode {mode}") + self.mode = mode + self.port = alloc_port() + self.connections = 0 + # Dual-stack: Warpgate resolves the target's hostname and dials the + # first address it gets, which for `localhost` is usually `::1`. A + # v4-only listener is simply never reached, and the test then passes for + # having tested nothing. + self._socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._socket.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + self._socket.bind(("::", self.port)) + self._socket.listen(8) + self._stop = threading.Event() + self._thread = threading.Thread(target=self._serve, daemon=True) + + def start(self): + self._thread.start() + + def stop(self): + self._stop.set() + try: + self._socket.close() + except OSError: + pass + self._thread.join(timeout=5) + + def _serve(self): + while not self._stop.is_set(): + try: + client, _ = self._socket.accept() + except OSError: + return + self.connections += 1 + threading.Thread( + target=self._handle, args=(client,), daemon=True + ).start() + + def _handle(self, client: socket.socket): + try: + client.settimeout(30) + if self.mode == "instant_close": + client.close() + return + + if self.mode == "banner_never_ends": + # No CR LF, ever. + while not self._stop.is_set(): + client.sendall(b"SSH-2.0-Endless" + b"A" * 1024) + return + + if self.mode == "banner_gigantic": + client.sendall(b"SSH-2.0-Huge" + b"B" * (8 * 1024 * 1024) + b"\r\n") + return + + client.sendall(b"SSH-2.0-Hostile_1.0\r\n") + + if self.mode == "silent_after_banner": + while not self._stop.is_set(): + self._stop.wait(0.5) + return + + if self.mode == "garbage_after_banner": + client.sendall(bytes(range(256)) * 64) + return + + if self.mode == "absurd_packet_length": + # A binary packet header claiming ~4 GiB of payload. A reader + # that reserves the declared length before checking it against + # anything allocates that much. + client.sendall(b"\xff\xff\xff\xf0" + b"\x00" * 16) + while not self._stop.is_set(): + self._stop.wait(0.5) + return + except OSError: + return + finally: + try: + client.close() + except OSError: + pass diff --git a/tests/mutation_matrix.py b/tests/mutation_matrix.py new file mode 100644 index 000000000..fe7c3730a --- /dev/null +++ b/tests/mutation_matrix.py @@ -0,0 +1,1616 @@ +"""Turn each guard off in turn and record which test notices. + +Written after a reviewer found several tests that passed without exercising what +they were named for. Checking tests one at a time, by hand, is how that happens: +you verify the test you just wrote and never ask which *other* test would have +caught the same thing, or whether any test covers a guard nobody thought to +break. + +Two things come out of a run. A guard that no test catches is a hole. A test +that never fails for any mutation is doing less than its name suggests — not +proof that it is useless, but the place to look next. + +Not a pytest module: it rebuilds the gateway between runs, so it is a script. + +Two modes, and the routine one is `--named`. From the repository root: + + PYTHONPATH=$PWD poetry -C tests run python -m tests.mutation_matrix --named + PYTHONPATH=$PWD poetry -C tests run python -m tests.mutation_matrix --named principal + ... --named --changed # only guards whose anchor file changed + ... --named --fail-fast # stop at the first guard that does not + # discriminate, rather than spending hours + # confirming the rest + +`PYTHONPATH` is not decoration. The virtualenv lives in `tests/`, so poetry has +to be pointed there, and `poetry -C tests run` executes with the working +directory *already changed* to `tests/` — at which point the `tests` package +this module belongs to is no longer importable and `-m` fails outright. A +relative `PYTHONPATH=.` resolves against that changed directory and fails the +same way. This docstring documented the shorter command for two rounds and +nobody ran it. + +`--named` is the A/B the coverage number comes from: for each guard it runs only +the test named after that guard, twice — once with the guard disabled, once with +it restored — and the guard discriminates when the test fails in the first run +and passes in the second. Two runs of one test per guard. + +Without the flag it asks a broader and far more expensive question — which of +*all* the tests notice the mutation — by rerunning the whole integration suite +and every crate's unit tests once per guard. That is hours, and it is the mode to +reach for when a guard has no named discriminator yet and you want to find out +what does catch it. It is not the routine invocation, and reviewers have run it +by mistake because this docstring used to name it first. +""" + +import ast +import atexit +import json +import pathlib +import signal +import subprocess +import sys +import time +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent + +# Crates whose unit tests can discriminate a guard — derived from the guards +# themselves rather than typed out. +# +# It was a hand-written tuple twice over: first two copies that drifted apart, +# then one copy that was simply incomplete. A guard was moved into +# `warpgate-web-ssh`, its test went with it, and the verifier could not see the +# test because nobody thought to extend the list. It refused rather than +# guessing, which is the only reason this is a footnote and not another false +# verdict — but a list that has to be remembered will be forgotten again. +# +# Every mutation names a file, and the crate is its first path segment. A guard +# in a crate therefore brings that crate with it. +def _crates_from_mutations() -> tuple[str, ...]: + seen = {path.split("/")[0] for _, path, _, _ in MUTATIONS} + # `warpgate` is the binary; its unit tests are not where guards are pinned. + return tuple(sorted(seen - {"warpgate"})) + +# Files currently rewritten in place, and their originals. A marker beside them +# so anything else touching this tree can tell — the pre-commit hook refuses to +# commit while it exists. +IN_FLIGHT: dict[str, str] = {} +LOCK = REPO / "tests" / ".matrix-running" + +# Each mutation names a guard and the edit that disables it. The replacement has +# to compile — a mutation that fails to build proves nothing about the tests. +MUTATIONS = [ + ( + "certificate: key ID must match", + "warpgate-protocol-ssh/src/client/mod.rs", + "if certificate.key_id() != key_id {", + "if false {", + ), + ( + "certificate: must be a user certificate", + "warpgate-protocol-ssh/src/client/mod.rs", + "if certificate.cert_type() != CertType::User {", + "if false {", + ), + ( + "certificate: must certify our ephemeral key", + "warpgate-protocol-ssh/src/client/mod.rs", + "if certificate.public_key() != key.key_data() {", + "if false {", + ), + ( + "certificate: principals must be exactly the target account", + "warpgate-protocol-ssh/src/client/mod.rs", + "if principals.len() != 1 || principals.first().is_none_or(|only| only != principal) {", + "if false {", + ), + ( + "certificate: pinned critical options must be present", + "warpgate-protocol-ssh/src/client/mod.rs", + "if expected.value.is_some() && !certificate.critical_options().contains_key(&expected.name)\n {", + "if false\n {", + ), + ( + # The complement of the entry above: pinning must not become mandatory + # for bare names too, or a role that sets an option only sometimes + # cannot be configured at all. + "certificate: a bare name permits without requiring", + "warpgate-protocol-ssh/src/client/mod.rs", + "if expected.value.is_some() && !certificate.critical_options().contains_key(&expected.name)\n {", + "if !certificate.critical_options().contains_key(&expected.name)\n {", + ), + ( + # Warpgate has no `valid_after` check — the refusal is the target's — + # so what is guarded here is the diagnostic that sends the operator to + # the clock rather than to the credentials. + "certificate: a target refusal names the validity window", + "warpgate-protocol-ssh/src/client/mod.rs", + # Anchored on the whole call, not on the message alone. Replacing just + # the string dropped its `{}` while `format!` still passed + # `validity.0, validity.1`, so the mutation did not compile — and a + # mutation that does not build measures nothing, which is the rule + # stated at the top of this file and broken twice now (W-103, W-107). + # The placeholders are kept and fed empty strings instead: the window + # leaves the message, the code still builds. + ''' "Certificate authentication was rejected by the SSH target \\ + (the certificate was valid from {} to {}; check the target\'s clock)", + validity.0, validity.1''', + ''' "Certificate authentication was rejected by the SSH target{}{}", + "", ""''', + ), + ( + "certificate: unexpected extensions refused", + "warpgate-protocol-ssh/src/client/mod.rs", + " if !allowed_extensions.iter().any(|allowed| allowed == name) {", + " if false {", + ), + ( + "certificate: unexpected critical options refused", + "warpgate-protocol-ssh/src/client/mod.rs", + " if named.peek().is_none() {", + " if false {", + ), + # Three arms, three guards. This was one entry anchored on an `&&` chain + # that no longer exists — the code became a match, the anchor stopped + # matching, and the script reported the guard as covered while never once + # disabling it. That is why a missing anchor is now fatal. + ( + "certificate: a never-expiring certificate is refused", + "warpgate-protocol-ssh/src/client/mod.rs", + " if certificate.valid_before() == u64::MAX {", + " if false {", + ), + ( + "certificate: an unrepresentable expiry is refused", + "warpgate-protocol-ssh/src/client/mod.rs", + """ None => { + return Some( + "Vault issued a certificate with an unrepresentable expiry time".to_owned(), + ); + }""", + " None => {}", + ), + ( + # Emptying the arm rather than guarding it with `false`: a guarded arm + # leaves the match non-exhaustive, and a mutation that does not compile + # measures nothing. + "certificate: an already-expired certificate is refused", + "warpgate-protocol-ssh/src/client/mod.rs", + """ Some(Err(_)) => { + return Some( + "Vault issued a certificate that has already expired; check the clock on this host" + .to_owned(), + ); + }""", + " Some(Err(_)) => {}", + ), + ( + "certificate: the returned window must match what was asked for", + "warpgate-protocol-ssh/src/client/mod.rs", + " if requested_ttl.is_some_and(|ttl| lifetime > ttl + CERTIFICATE_TTL_SLACK) =>", + " if false =>", + ), + ( + "certificate: lifetime is bounded", + "warpgate-protocol-ssh/src/client/mod.rs", + " Some(Ok(lifetime)) if lifetime > MAX_CERTIFICATE_LIFETIME => {", + " Some(Ok(lifetime)) if false && lifetime > MAX_CERTIFICATE_LIFETIME => {", + ), + ( + "connection: an untrusted jump host is refused, not traversed", + "warpgate-protocol-ssh/src/client/mod.rs", + " return Err(ConnectionError::UntrustedJumpHost);", + " {}", + ), + ( + "connection: a host-key check stops before authenticating", + "warpgate-protocol-ssh/src/client/mod.rs", + # Anchored on the predicate, not on the call site, so a unit test can + # decide it. The call-site mutation was pinned by an integration test + # asking a different question entirely, and measured as not + # discriminating — a real sshd refuses on its own, so "stopped after the + # key" and "carried on and got refused" look identical end to end. + " matches!(self, Self::CheckedHost)", + " false", + ), + ( + "web-ssh: connection errors are sanitised before the user sees them", + "warpgate-web-ssh/src/manager.rs", + # Anchored on the named boundary rather than on a call inside the event + # loop. A test cannot stand at that call without driving a browser + # session, and the integration test credited with covering this guard + # turned out to exercise the SSH path instead — measured, not suspected. + " error.client_message()", + " error.to_string()", + ), + ( + "vault: certificate_ttl outside the allowed range is refused at config load", + "warpgate-vault/src/client.rs", + " return Err(VaultError::InvalidCertificateTtl(ttl));", + " {}", + ), + ( + # The upper half specifically: the lower bound was checked and the + # ceiling was not, so a mutation that drops only the ceiling has to be + # visible on its own. + "vault: certificate_ttl above the ceiling is refused at config load", + "warpgate-vault/src/client.rs", + "&& (ttl.as_secs() == 0 || ttl > MAX_CERTIFICATE_LIFETIME)", + "&& ttl.as_secs() == 0", + ), + ( + "connection: the inter-hop tunnel open is bounded", + "warpgate-protocol-ssh/src/client/mod.rs", + " let channel = tokio::time::timeout(\n HANDSHAKE_TIMEOUT,", + " let channel = tokio::time::timeout(\n Duration::from_secs(86400),", + ), + ( + # The split. Collapsing authentication back onto the transport + # handshake's constant is the shape it had, and a Vault slow enough to + # exceed 30s then reads as the target failing to finish a handshake. + "connection: authentication has its own budget", + "warpgate-protocol-ssh/src/client/mod.rs", + # Disabled by neutralising the arithmetic, not by guarding the arm. + # The previous replacement renamed the binding to `_per_call` while the + # arm body still used `per_call`, so it did not compile — and a mutation + # that does not build measures nothing, which is this file's own rule. + # `cargo test` then produced no test output at all, and the verifier + # reported "could not run" for three runs before anyone read the build + # error. The matrix's `run_one` would have called it "did not compile", + # which is also not "caught"; two full runs had ended on the canary + # before reaching it, so nobody saw either. + " (SSHTargetAuth::Certificate(_), Some(per_call)) => AUTHENTICATION_TIMEOUT\n .max(per_call * VAULT_CALLS_PER_AUTHENTICATION + Duration::from_secs(5)),", + " (SSHTargetAuth::Certificate(_), Some(per_call)) => AUTHENTICATION_TIMEOUT\n .max(per_call * 0 + Duration::from_secs(0)),", + ), + ( + "connection: handshake deadline", + "warpgate-protocol-ssh/src/client/mod.rs", + "let handshake_deadline = tokio::time::sleep(HANDSHAKE_TIMEOUT);", + "let handshake_deadline = tokio::time::sleep(Duration::from_secs(86400));", + ), + ( + # The pin must survive a duplicate name. `.find()` took whichever entry + # the operator typed first, so a bare row beside a pinned one cancelled + # the pin silently while the admin UI still showed it. + "certificate: every matching pin is enforced, not the first", + "warpgate-protocol-ssh/src/client/mod.rs", + " for option in named {", + " for option in named.take(1) {", + ), + ( + # A token is not a person, and a person may not take a token's name. + "users: a token attribution cannot be claimed as a username", + "warpgate-admin/src/api/users.rs", + # Repointed after rustfmt folded the three conditions onto one line. + "&& !TOKEN_ATTRIBUTIONS.contains(&username)", + "&& !TOKEN_ATTRIBUTIONS.contains(&\"nobody-has-this-name\")", + ), + ( + # The other half of the attribution claim. `key_id_field` sanitises a + # username on its way into the certificate; this refuses to create one + # that would need sanitising. It shipped with no test in either language + # and no entry here — the only check in this feature with neither. + "users: a username cannot contain the key ID separator", + "warpgate-admin/src/api/users.rs", + "&& !username.contains(':')", + "&& !username.is_empty()", + ), + ( + # The deadline is paused while a host key is being decided on. This is + # the line that ends the pause. Only the arming line above was guarded, + # so the pause could be — and for a week was — permanent, and the matrix + # reported the deadline as covered. + # Repointed at the policy rather than at the line that applies it. + # The integration test that used to be named here never reached the + # line: the stalling fixture mutes before `NEWKEYS`, and russh does not + # call `check_server_key` until the exchange completes, so the pause and + # the resume were both dead code in that test. Measured twice — see + # W-116 — rather than argued. + "connection: the handshake deadline resumes after a host key answer", + "warpgate-protocol-ssh/src/client/mod.rs", + # Anchored on the call, not on the constant. The constant was the anchor + # while the test that discriminated it compared two constants, so the + # pair agreed with each other and nothing established that either was + # ever called. + """fn resume_after_host_key_answer(deadline: Pin<&mut tokio::time::Sleep>) { + deadline.reset(tokio::time::Instant::now() + once_the_host_key_is_answered()); +}""", + """fn resume_after_host_key_answer(deadline: Pin<&mut tokio::time::Sleep>) { + deadline.reset(tokio::time::Instant::now() + while_a_host_key_answer_is_outstanding()); +}""", + ), + ( + "certificate: a username cannot shift the key ID fields", + "warpgate-protocol-ssh/src/client/mod.rs", + # Repointed three times: when the attribution substitution joined this + # function, when it left for `user_key_id_field` because it was renaming + # the gateway itself, and when the substitution became a percent + # encoding because `root:admin` and `root_admin` collided under it. + # Each move was caught by `check_anchors` and by nothing else — a guard + # whose anchor has gone stale is reported as measured while never once + # being disabled, which is the failure this whole file exists to stop. + """ name.replace('%', "%25").replace(':', "%3A")""", + """ name.to_owned()""", + ), + ( + "certificate: a host-key check names the admin who asked", + "warpgate-protocol-ssh/src/client/mod.rs", + # Repointed when the hint stopped being a bare string: the gateway's own + # attribution and a person's name are now carried apart. + " None => self.identity_hint.as_ref().map(|hint| match hint {", + " None => None.map(|hint: &IdentityHint| match hint {", + ), + ( + # And names it honestly: a token is not a person, and the first fix + # recorded one as though it were. + "auth: a token is not attributed as a person", + "warpgate-common-http/src/auth.rs", + 'pub const TOKEN_ATTRIBUTIONS: [&str; 2] = ["admin-token", "cluster-token"];', + 'pub const TOKEN_ATTRIBUTIONS: [&str; 2] = ["admin", "cluster"];', + ), + ( + # The identity half. Reverting `role` to "is this the last hop" is the + # shape the code had, and it gives the same answer for every chain built + # today — which is why nothing noticed. + "host key: the hop is chosen by identity, not by position", + "warpgate-protocol-ssh/src/client/mod.rs", + " Some(asked_about) if asked_about == hop_id => HopRole::CheckedHost,", + " Some(_) => HopRole::CheckedHost,", + ), + ( + "host key: only the hop that was asked about reports", + "warpgate-protocol-ssh/src/client/mod.rs", + "matches!(self, Self::Connecting | Self::CheckedHost)", + "true", + ), + ( + "vault: principal must be one harmless entry", + "warpgate-vault/src/client.rs", + "if principal.is_empty() || principal.contains(',') || principal.chars().any(char::is_control) {", + "if false {", + ), + ( + "vault: key ID must not carry control characters", + "warpgate-vault/src/client.rs", + "if key_id.chars().any(char::is_control) || key_id.len() > MAX_KEY_ID {", + "if key_id.len() > MAX_KEY_ID {", + ), + ( + "vault: key ID length is bounded", + "warpgate-vault/src/client.rs", + "if key_id.chars().any(char::is_control) || key_id.len() > MAX_KEY_ID {", + "if key_id.chars().any(char::is_control) {", + ), + ( + # Repointed when the rule moved to `warpgate-common`, so the admin API + # could refuse at save time by the same test the signing path applies. + "vault: mount and role stay one path segment", + "warpgate-common/src/config/target.rs", + " && name\n .chars()\n .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-')", + " && true", + ), + ( + # No discriminating test yet, and the matrix says so rather than the + # guard being absent from it: verified by an A/B on the built binary — + # `create-user` against a namespaced mount fails with "invalid Vault + # role or mount name" through `Services::new` and succeeds through + # `new_without_vault`. `recover-access` takes the same path but asserts + # an interactive terminal first, so it cannot be driven from a test + # harness as it stands. + "commands: break-glass does not depend on Vault", + "warpgate/src/commands/create_user.rs", + "Services::new_without_vault(config.clone(), None, params.clone())", + "Services::new(config.clone(), None, params.clone())", + ), + ( + "vault: an unbound AWS login is called out", + "warpgate-vault/src/client.rs", + " VaultAuth::Aws {\n server_id: None, ..\n } => Some(", + " VaultAuth::Aws {\n server_id: None, ..\n } if false => Some(", + ), + ( + "vault: address must be HTTPS or loopback", + "warpgate-vault/src/client.rs", + "if !is_loopback {", + "if false {", + ), + ( + "vault: redirects are refused", + "warpgate-vault/src/client.rs", + "let mut builder = reqwest::Client::builder()\n .redirect(reqwest::redirect::Policy::none())\n .timeout(config.timeout);", + "let mut builder = reqwest::Client::builder()\n .timeout(config.timeout);", + ), + ( + "vault: response bodies are bounded", + "warpgate-vault/src/client.rs", + "if buf.len() + chunk.len() > MAX_RESPONSE_BODY {", + "if false {", + ), + ( + "vault: absurd lease refused rather than panicking", + "warpgate-vault/src/client.rs", + ".ok_or(VaultError::InvalidLease(seconds))?,", + ".unwrap_or_else(|| Instant::now()),", + ), + # `vault: credential file size is capped` used to be here and is not a + # guard. Measured, not argued: with the check disabled, + # `an_oversized_regular_file_is_refused` still passes, because the stream + # bound eight lines below refuses the same file. It has no input of its + # own — `read_credential` stats the open handle rather than the path, so + # there is no window in which the two could disagree, and a FIFO, the one + # source that lies about its size, is caught by the stream bound alone. + # + # So it is an early-out that avoids reading 16 KB before refusing, and + # listing it as a security guard inflated the count by one while + # guaranteeing a permanent failure in any honest verification. Removed + # rather than left failing: the entry was a classification error, and + # keeping a non-guard in the list to avoid appearing to remove a failure + # would be the same dishonesty pointed the other way. + ( + # The half the stat could not see: what actually arrives, from the + # handle already open, rather than what the filesystem claimed. + "vault: the credential stream itself is bounded", + "warpgate-vault/src/client.rs", + " if read > MAX_CREDENTIAL_FILE {", + " if false {", + ), + ( + "vault: wrapping token redeemed once", + "warpgate-vault/src/client.rs", + "if let Some(entry) = cached.as_ref()\n && entry.source.as_str() == cred.as_str()", + "if let Some(entry) = cached.as_ref()\n && false", + ), + ( + # The gateway's own attribution is reachable as a username through five + # of the six paths that create one. Held where the key ID is built, so + # which path created the user stops mattering. + "certificate: a username cannot impersonate the gateway's attribution", + "warpgate-protocol-ssh/src/client/mod.rs", + 'if is_reserved_key_id_field(&field) {\n return format!("{field}_");\n }', + 'if false {\n return format!("{field}_");\n }', + ), + ( + # Ours, found by unskipping the integration test in round J. Rendering + # the validity window for the diagnostic panicked on a never-expiring + # certificate, in a tokio worker, before the check that refuses one. + "certificate: describing a far-future expiry cannot panic", + "warpgate-protocol-ssh/src/client/mod.rs", + """ let mut rendered = String::new(); + if write!(rendered, "{}", humantime::format_rfc3339_seconds(at)).is_err() { + return "a date beyond any this can render".to_owned(); + } + rendered""", + """ humantime::format_rfc3339_seconds(at).to_string()""", + ), + ( + # Raised externally, round J: an operator checking a host key saw + # `SSH protocol error` both for a host that could not be reached and for + # one whose key is not trusted — one sentence for two different jobs. + "connection: unreachable and untrusted do not read alike", + "warpgate-protocol-ssh/src/client/mod.rs", + """ ConnectionError::Io(e) | ConnectionError::Ssh(russh::Error::IO(e)) => { + format!( + "Could not open an SSH connection to the target: {}", + unreachable_reason(e.kind()) + ) + }""", + """ ConnectionError::Io(_) | ConnectionError::Ssh(russh::Error::IO(_)) => { + "SSH protocol error".to_string() + }""", + ), + ( + # Raised externally, round J: the sanitiser had no test at all, and the + # nearest one asserts a string that appears in an unrelated variant's + # `Display` too, so it passes with the sanitising removed. + "error surfacing: internal error text never reaches a client message", + "warpgate-protocol-ssh/src/client/mod.rs", + """ ConnectionError::Warpgate(_) => "Internal connection error".to_string(),""", + """ ConnectionError::Warpgate(e) => e.to_string(),""", + ), + ( + # Raised externally, round J: the whole authentication step was bounded + # by a budget that grows with `vault.timeout`, so a target that went + # quiet after receiving its certificate was waited on for as long as + # Vault was allowed to be slow. + "connection: the target's own USERAUTH answer is bounded", + "warpgate-protocol-ssh/src/client/mod.rs", + """ tokio::time::timeout(bound, what) + .await + .map_err(|_| ConnectionError::TargetAuthenticationTimeout)? + .map_err(ConnectionError::from)""", + """ what.await.map_err(ConnectionError::from)""", + ), + ( + # Raised externally, round J: the substitution mapped `root:admin` and + # `root_admin` onto one field, so the log line this feature exists to + # produce could name a person who did not connect. + "certificate: two usernames cannot produce one key ID", + "warpgate-protocol-ssh/src/client/mod.rs", + """ name.replace('%', "%25").replace(':', "%3A")""", + """ name.replace(':', "_")""", + ), + ( + # The reserved names lived in two places and only one was consulted. + # `UNATTRIBUTED` goes into the same field, written by the same code, and + # a user of that name read as a session with no user recorded at all. + "certificate: the unattributed placeholder cannot be claimed by a user", + "warpgate-protocol-ssh/src/client/mod.rs", + """ TOKEN_ATTRIBUTIONS.contains(&field) || field == UNATTRIBUTED""", + """ TOKEN_ATTRIBUTIONS.contains(&field)""", + ), + ( + # Validated at connect time already; the guard is that the admin API + # refuses the same name at save time, where the operator can fix it. + "admin: a Vault role the signing path would refuse is refused on save", + "warpgate-admin/src/api/targets.rs", + " .is_none_or(|role| warpgate_common::vault_name_is_well_formed(role))", + " .is_none_or(|_| true)", + ), + ( + # The one response-side property that had no check at all: every other + # asks whether the certificate matches the request, none asked who + # signed it. + "certificate: the signing CA must be the pinned one", + "warpgate-protocol-ssh/src/client/mod.rs", + " if certificate.signature_key() == expected.key_data() {\n return None;\n }", + " if true {\n return None;\n }", + ), + ( + "connection: a chain missing the host asked about is refused", + "warpgate-protocol-ssh/src/client/mod.rs", + " check_target.is_none_or(|asked_about| hops.contains(&asked_about))", + " check_target.is_none_or(|asked_about| asked_about == asked_about)", + ), +] + +# A mutation nothing can possibly catch: the text of a debug log line no test +# asserts on. It must come back SURVIVED. +# +# If it comes back caught, some test in the suite is failing for a reason +# unrelated to whatever is being mutated — a flake, a stale build, a broken +# fixture — and every "caught" verdict in that run is worthless, because the +# script cannot tell a guard doing its job from a suite that fails no matter +# what. The matrix exists to stop us trusting tests we have not checked; it +# needs the same treatment applied to itself. +CANARY = ( + "canary: an inert change no test can see", + "warpgate-vault/src/client.rs", + '"Authenticated to Vault"', + '"Authenticated to Vault (canary)"', +) + +# Which test is supposed to be the one that notices. +# +# "Some test failed" and "this guard is covered" are different claims, and this +# suite is built on top of a real sshd that enforces most of what we check for +# itself. Break the expiry guard and the connection still fails — the target +# refuses the expired certificate on its own. Break the principal guard and the +# target refuses an account it does not have. Break the unexpected-options guard +# with an option name OpenSSH does not recognise and the target refuses that too. +# In every one of those the run records "caught" and the guard is not covered by +# anything: deleting it changes nothing a test can see. +# +# So a guard is covered only when the test named after it is among the failures. +# For an integration test that means asserting on Warpgate's *own* refusal +# message, which needs a PTY — a test that only checks a non-zero exit code +# cannot tell our refusal from the target's. +# +RUST_CRATES = _crates_from_mutations() + +# A guard with no entry here is reported, not skipped. Not knowing which test +# discriminates a guard is the same state as not having one. +DISCRIMINATES = { + "certificate: key ID must match": ["test_a_certificate_with_a_different_key_id_is_refused"], + "connection: the handshake deadline resumes after a host key answer": [ + "answering_a_host_key_question_puts_the_targets_own_bound_back" + ], + # The seventeen that had no entry. Most already had a discriminating test — + # it had simply never been written down, which under the criterion in §8 is + # the same state as having none: nothing established that the test noticing + # was the test named after the guard. + "certificate: must be a user certificate": [ + "test_a_host_certificate_is_not_offered_to_the_target" + ], + # Its own test since W-119. It shared one with the two host-key guards + # below, which need the opposite starting condition — a jump host that *is* + # trusted — so one test was setting up both worlds and the refusal it + # asserted was only ever the first half of a longer story. + "connection: an untrusted jump host is refused, not traversed": [ + "test_an_untrusted_jump_host_is_refused_rather_than_traversed" + ], + "connection: a host-key check stops before authenticating": [ + "a_host_key_check_stops_at_the_hop_it_asked_about", + "each_role_reports_and_stops_as_its_name_says", + ], + "connection: the inter-hop tunnel open is bounded": [ + "test_a_jump_host_that_never_opens_the_tunnel_is_given_up_on" + ], + "connection: authentication has its own budget": [ + "a_certificate_target_gets_a_budget_that_fits_its_vault_calls" + ], + "vault: principal must be one harmless entry": ["test_principal_validation"], + "vault: key ID must not carry control characters": ["test_key_id_validation"], + # Same source line as the entry above, opposite half. `test_key_id_validation` + # had no length case until this round, so a mutation dropping only the bound + # was caught by nothing while the line looked covered. + "vault: key ID length is bounded": ["test_key_id_validation"], + "vault: mount and role stay one path segment": ["test_segment_validation"], + "commands: break-glass does not depend on Vault": [ + "test_break_glass_user_creation_does_not_depend_on_vault" + ], + "vault: an unbound AWS login is called out": ["an_unbound_aws_login_is_called_out"], + "vault: address must be HTTPS or loopback": ["test_address_validation"], + "vault: redirects are refused": [ + "test_a_redirect_never_carries_the_token_to_another_host" + ], + "vault: response bodies are bounded": [ + "test_an_oversized_success_body_is_refused_rather_than_buffered" + ], + "vault: absurd lease refused rather than panicking": [ + "test_an_absurd_lease_is_refused_rather_than_panicking" + ], + # One test each, because one test could not do both. The caveat that used + # to sit here claimed this entry "does discriminate the line the guard + # names"; `verify_named_rust` measured that claim and it was false. An + # oversized regular file is refused by the `stat` early-out and by the + # stream bound alike, so disabling either left the other to catch it and + # neither guard was evidenced. A FIFO reports a size of zero, which is the + # only input the stream bound has to itself. + "vault: the credential stream itself is bounded": [ + "only_the_stream_bound_can_refuse_a_source_that_lies_about_its_size" + ], + "vault: wrapping token redeemed once": [ + "test_a_wrapping_token_is_redeemed_once_and_the_secret_id_reused" + ], + "certificate: every matching pin is enforced, not the first": [ + "a_bare_duplicate_does_not_cancel_a_pinned_value", + "conflicting_pins_refuse_everything_rather_than_picking_one", + ], + "users: a token attribution cannot be claimed as a username": [ + "a_username_with_a_colon_would_shift_every_field_of_the_key_id" + ], + "users: a username cannot contain the key ID separator": [ + "a_username_with_a_colon_would_shift_every_field_of_the_key_id" + ], + "connection: handshake deadline": [ + "test_a_target_that_stalls_the_handshake_is_given_up_on" + ], + "certificate: must certify our ephemeral key": [ + "test_certificate_issued_for_a_key_warpgate_does_not_hold" + ], + "certificate: a never-expiring certificate is refused": [ + "a_never_expiring_certificate_is_refused" + ], + "certificate: an unrepresentable expiry is refused": ["an_unrepresentable_expiry_is_refused"], + "certificate: an already-expired certificate is refused": [ + "an_already_expired_certificate_is_refused", + "test_expired_certificate", + ], + "certificate: a target refusal names the validity window": [ + "test_certificate_that_is_not_yet_valid" + ], + "certificate: a username cannot impersonate the gateway's attribution": [ + "a_username_cannot_impersonate_the_gateways_own_attribution" + ], + "certificate: describing a far-future expiry cannot panic": [ + "a_far_future_expiry_is_described_rather_than_panicked_on" + ], + "connection: unreachable and untrusted do not read alike": [ + "an_unreachable_target_does_not_read_like_an_untrusted_key" + ], + "error surfacing: internal error text never reaches a client message": [ + "no_internal_error_text_reaches_a_client_message" + ], + "connection: the target's own USERAUTH answer is bounded": [ + "a_target_that_never_answers_userauth_is_given_up_on" + ], + "certificate: two usernames cannot produce one key ID": [ + "two_usernames_cannot_collide_in_a_key_id" + ], + "certificate: the unattributed placeholder cannot be claimed by a user": [ + "a_username_cannot_impersonate_the_unattributed_placeholder" + ], + "admin: a Vault role the signing path would refuse is refused on save": [ + "a_role_the_signing_path_would_refuse_is_refused_at_save_time" + ], + "certificate: the signing CA must be the pinned one": [ + "a_certificate_from_an_unpinned_ca_is_refused" + ], + "connection: a chain missing the host asked about is refused": [ + "a_chain_without_the_host_asked_about_cannot_answer" + ], + "certificate: lifetime is bounded": ["a_certificate_outliving_the_bound_is_refused"], + "certificate: the returned window must match what was asked for": [ + "a_certificate_longer_than_the_requested_ttl_is_refused" + ], + "certificate: principals must be exactly the target account": [ + "test_a_certificate_naming_the_wrong_account_is_refused", + "test_certificate_for_a_different_principal", + ], + "certificate: pinned critical options must be present": [ + "a_pinned_option_missing_from_the_certificate_is_refused" + ], + "certificate: a bare name permits without requiring": [ + "an_option_permitted_by_name_may_be_absent" + ], + "vault: certificate_ttl outside the allowed range is refused at config load": [ + "a_certificate_ttl_outside_the_allowed_range_is_refused_at_construction" + ], + "vault: certificate_ttl above the ceiling is refused at config load": [ + "a_certificate_ttl_outside_the_allowed_range_is_refused_at_construction" + ], + "certificate: unexpected extensions refused": [ + "an_extension_the_target_did_not_name_is_refused" + ], + "certificate: unexpected critical options refused": [ + "an_option_the_target_did_not_name_is_refused", + "test_an_unexpected_forced_command_is_refused", + ], + "certificate: a username cannot shift the key ID fields": [ + "a_username_carrying_a_colon_cannot_shift_the_key_id_fields" + ], + "auth: a token is not attributed as a person": [ + "test_checking_a_chained_target_authenticates_only_to_the_jump_host" + ], + "certificate: a host-key check names the admin who asked": [ + "test_checking_a_chained_target_authenticates_only_to_the_jump_host" + ], + # These two do share a discriminator, and it was looked at rather than left + # alone (W-119). They are one decision read two ways — which hop is the + # answer, and which hops may speak — and on a chain of two every mutation of + # either ends in the same observation: the endpoint answers with the jump + # host's key. Separating them end to end would take an observation neither + # the endpoint nor the target makes; the unit test that pins the decision + # itself, `each_role_reports_and_stops_as_its_name_says`, is equally unable + # to tell them apart, and for the same reason. + "host key: the hop is chosen by identity, not by position": [ + "test_the_host_key_check_reports_the_target_and_not_the_jump_host" + ], + "host key: only the hop that was asked about reports": [ + "test_the_host_key_check_reports_the_target_and_not_the_jump_host" + ], + "web-ssh: connection errors are sanitised before the user sees them": [ + "a_browser_never_sees_the_error_s_own_words" + ], +} + +# Bare names, run from `tests/`: poetry changes into its project directory, so +# a path relative to the repository root finds nothing — and pytest reports +# "no tests ran" rather than failing, which reads exactly like every guard +# surviving. The first run of this script did precisely that. +SUITES = [ + "test_ssh_target_cert_auth.py", + "test_vault_hostile_certs.py", + "test_vault_hostile_target.py", +] + + +def run(command, **kwargs): + return subprocess.run(command, cwd=REPO, capture_output=True, text=True, **kwargs) + + +def run_named_only(tests: list[str], crates=None) -> tuple[set[str], set[str]]: + """Run only the given tests. Returns (passed, failed), by name. + + The whole suite is not run. That is the point: `run_one` runs everything + because it asks "which tests notice", and the answer to that question is + worth an hour only while the guard has no named discriminator. Once it has + one, the question in §8 is narrower — does *this* test notice — and one test + answers it. + + It is also the way past W-25b. Two full runs ended on the canary with five + unrelated failures that passed again in isolation, so the suite is not + stable under the load a full sweep creates. Running one test creates no such + load, and the A/B below is decisive on its own: the named test must pass + before the mutation and fail after it. A test that was already failing + cannot be mistaken for a guard being caught, because the "before" half + catches that first. + """ + passed: set[str] = set() + failed: set[str] = set() + + crates = crates or list(RUST_CRATES) + rust = [t for t in tests if not t.startswith("test_") or _is_rust(t, crates)] + python = [t for t in tests if t not in rust] + + # Stops as soon as every wanted test has been located. Without this, a guard + # whose one discriminator lives in the first crate still built the test + # binary of every other crate, to ask each whether it also owned a name that + # had already been found. + outstanding = set(rust) + for crate in crates: + if not outstanding: + break + wanted = [t for t in outstanding if t in _crate_tests(crate)] + if not wanted: + continue + outstanding -= set(wanted) + for name in wanted: + result = run(["cargo", "test", "-p", crate, "--", "--exact", *_paths(crate, name)]) + (failed if result.returncode != 0 else passed).add(name) + + if python: + result = subprocess.run( + ["poetry", "run", "pytest", *SUITES, "-q", "--tb=no", "-p", "no:randomly", + "-k", " or ".join(python)], + cwd=REPO / "tests", + capture_output=True, + text=True, + ) + reported = { + line.split("::")[-1].split()[0] + for line in result.stdout.splitlines() + if line.startswith("FAILED") + } + for name in python: + (failed if name in reported else passed).add(name) + + return passed, failed + + +_RUST_TEST_CACHE: dict[str, dict[str, str]] = {} + + +def _crate_tests(crate: str) -> dict[str, str]: + """Test name to its full `module::path::name`, for `--exact`.""" + if crate not in _RUST_TEST_CACHE: + listed = run(["cargo", "test", "-p", crate, "--", "--list"]) + paths = {} + for line in listed.stdout.splitlines(): + if line.endswith(": test"): + path = line.rsplit(":", 1)[0].strip() + paths.setdefault(path.split("::")[-1], path) + _RUST_TEST_CACHE[crate] = paths + return _RUST_TEST_CACHE[crate] + + +def _paths(crate: str, name: str) -> list[str]: + path = _crate_tests(crate).get(name) + return [path] if path else [name] + + +def _crates_nearest(path: str) -> list[str]: + """`RUST_CRATES`, with the crate a guard lives in tried first. + + Listing a crate's tests builds that crate's test binary, so the order these + are consulted in is most of the cost of a small run. A guard's discriminator + is usually — not always — in the guard's own crate, so trying that one first + turns six builds into one in the common case while leaving the uncommon one + correct. + """ + own = path.split("/")[0] + if own not in RUST_CRATES: + return list(RUST_CRATES) + return [own] + [c for c in RUST_CRATES if c != own] + + +def _is_rust(name: str, crates=None) -> bool: + return any(name in _crate_tests(crate) for crate in (crates or RUST_CRATES)) + + +# Guards added or repointed in round J. A scheduling hint only: it puts the +# least-established guards first so a `--fail-fast` run reaches them in minutes +# rather than hours. Drifting out of date costs nothing here, unlike +# `DISCRIMINATES`, where a stale name is a guard that cannot be measured. +RECENT_GUARDS = frozenset({ + "certificate: two usernames cannot produce one key ID", + "certificate: the unattributed placeholder cannot be claimed by a user", + "certificate: a username cannot shift the key ID fields", + "certificate: a username cannot impersonate the gateway's attribution", + "certificate: describing a far-future expiry cannot panic", + "connection: the target's own USERAUTH answer is bounded", + "connection: unreachable and untrusted do not read alike", + "connection: the handshake deadline resumes after a host key answer", + "error surfacing: internal error text never reaches a client message", +}) + + +def _needs_gateway_binary(tests, crates) -> bool: + """Whether measuring this guard runs the gateway at all. + + A Rust discriminator is compiled and run by `cargo test -p `, which + builds its own test binary from the mutated source and never looks at + `target/debug/warpgate`. Building the gateway for those guards — 33 of the + 53 — was work whose result nothing read. + """ + return any(t.startswith("test_") and not _is_rust(t, crates) for t in tests) + + +def verify_named(mutations, fail_fast=False): + """A/B every guard against the test named after it, one at a time. + + Reports per guard: `discriminates`, `does not discriminate` (the named test + passed with the guard disabled, so it is not evidence for it), `already + failing` (the baseline was red, so nothing could be concluded), or `did not + compile`. + """ + results = [] + total = len(mutations) + started_at = time.time() + + def stamp() -> str: + """Elapsed since measurement began, as `h:mm:ss`. + + Wall-clock duration is the only thing anyone asks of a run this long — + how far in, how much left — and it was the one thing the output did not + say. Elapsed rather than a clock time, because the interesting quantity + is the cost of a guard, not the hour it happened to fall in. + """ + seconds = int(time.time() - started_at) + return f"{seconds // 3600}:{seconds // 60 % 60:02d}:{seconds % 60:02d}" + + def record(result): + """Appended *and printed*, as each guard finishes. + + Everything used to be printed after the loop returned, so a run said + nothing at all until it was over. A 34-guard sweep is well over an hour + here, and for that hour the only thing distinguishing progress from a + hang was `ps`. An instrument that cannot be told apart from a stuck one + gets killed, and two of mine were. + """ + results.append(result) + result["at"] = stamp() + print( + f"[{len(results):>2}/{total}] {result['status']:>26} " + f"{result['guard']} ({stamp()})", + flush=True, + ) + return result + + # `target/debug/warpgate` is left holding the last mutation applied to it, + # because restoring a source file does not rebuild the binary compiled from + # it. The next guard's *baseline* then ran the previous guard's mutated + # gateway, and reported `already failing` for a tree that was fine — twice + # in one sweep, on guards 29 and 31. Raised by Antigravity as W-152, and it + # invalidated the baseline half of every integration guard's A/B. + binary_is_mutated = False + + for index, (name, path, old, new) in enumerate(mutations, start=1): + expected = DISCRIMINATES.get(name) + if not expected: + print(f"[{index:>2}/{total}] measuring {name}", flush=True) + record({"guard": name, "status": "no discriminating test named"}) + continue + + nearest = _crates_nearest(path) + needs_binary = _needs_gateway_binary(expected, nearest) + kind = "integration" if needs_binary else "unit" + print(f"[{index:>2}/{total}] measuring {name} ({kind}, {stamp()})", flush=True) + + if needs_binary and binary_is_mutated: + print(f"[{index:>2}/{total}] rebuilding the gateway from clean source", flush=True) + run(["cargo", "build", "--bin", "warpgate"]) + binary_is_mutated = False + + print(f"[{index:>2}/{total}] baseline", flush=True) + before_pass, before_fail = run_named_only(expected, nearest) + if before_fail: + record({ + "guard": name, + "status": "already failing", + "tests": sorted(before_fail), + }) + continue + + source = REPO / path + original = source.read_text() + IN_FLIGHT[str(source)] = original + source.write_text(original.replace(old, new, 1)) + try: + if needs_binary: + print(f"[{index:>2}/{total}] building the gateway with the guard off", flush=True) + build = run(["cargo", "build", "--bin", "warpgate"]) + binary_is_mutated = True + if build.returncode != 0: + record({"guard": name, "status": "did not compile"}) + continue + print(f"[{index:>2}/{total}] testing with the guard off", flush=True) + after_pass, after_fail = run_named_only(expected, nearest) + # `all`, per A2: every test the entry names has to notice. + status = ( + "discriminates" + if set(expected) <= after_fail + else "does not discriminate" + ) + record({ + "guard": name, + "status": status, + "expected": expected, + "failed_with_guard_off": sorted(after_fail), + "passed_with_guard_off": sorted(after_pass), + }) + finally: + source.write_text(original) + IN_FLIGHT.pop(str(source), None) + + if fail_fast and results[-1]["status"] != "discriminates": + print( + f"\nstopping at [{index}/{total}] because --fail-fast was asked " + f"for and this guard came back {results[-1]['status']!r}.\n" + f"{len(results)} of {total} measured; the rest are unknown, not " + f"passing.", + flush=True, + ) + break + + # The tree is clean by now but the binary is not, and whatever runs next + # deserves one built from the source that is actually on disk. + if binary_is_mutated: + print("rebuilding the gateway from clean source", flush=True) + run(["cargo", "build", "--bin", "warpgate"]) + return results + + +def failing_tests() -> tuple[set[str], str]: + """Which tests fail right now. + + Refuses to report anything when the run collected nothing: a suite that did + not execute looks identical to a suite where every guard survived, and that + is the exact confusion this script exists to expose. + """ + result = subprocess.run( + ["poetry", "run", "pytest", *SUITES, "-q", "--tb=no"], + cwd=REPO / "tests", + capture_output=True, + text=True, + ) + if "no tests ran" in result.stdout or "collected 0 items" in result.stdout: + raise SystemExit(f"the suite collected nothing, so nothing was measured:\n{result.stdout[-500:]}") + failed = { + line.split("::")[-1].split()[0] + for line in result.stdout.splitlines() + if line.startswith("FAILED") + } + return failed, result.stdout[-400:] + + +def check_anchors(mutations): + """Every anchor must still match the source, before anything is run. + + An anchor that has drifted used to be recorded and stepped over, and the + final tally counted it alongside the guards that were genuinely caught — so + the script reported full coverage for a guard it had never once disabled. + A guard whose anchor no longer matches is not a guard that passed; it is a + guard that was never tested, and the run has to stop and say so. + """ + stale = [ + (name, path) + for name, path, old, _ in mutations + if old not in (REPO / path).read_text() + ] + if stale: + lines = "\n".join(f" {name}\n in {path}" for name, path in stale) + raise SystemExit( + f"{len(stale)} anchor(s) no longer match the source, so those guards " + f"cannot be measured:\n{lines}\n\n" + "Repoint them at the current code before trusting any number from " + "this script." + ) + + +def _package_of(crate_dir: str) -> str: + """The Cargo package name for a directory, read rather than assumed. + + They match in this workspace today. Reading it costs one line and removes a + convention from the set of things a future rename can quietly break. + """ + manifest = (REPO / crate_dir / "Cargo.toml").read_text() + for line in manifest.splitlines(): + if line.startswith("name"): + return line.split("=", 1)[1].strip().strip('"') + raise SystemExit(f"{crate_dir}/Cargo.toml names no package") + + +def check_replacements_build(mutations, in_flight): + """Every replacement must compile, before anything is measured. + + The rule is stated at the top of this file and has been broken twice. W-103 + renamed a binding the arm body still used; W-107 dropped the `{}` from a + format string that still passed two arguments. Both were found hours into a + run, when the guard's turn finally came and the build failed — and a guard + whose mutation does not build has not been measured, whatever the tally + says. + + Checked in as few passes as the matrix allows, not one per guard: the + replacements are applied together and the packages they touch are checked + in one `cargo check`, so the whole matrix costs two builds rather than + forty-seven. The binary crate is included even though it holds no + discriminating tests, because "it compiles" is a claim about all of the + code, not the tested part. + + More than one pass is needed because some guards deliberately share an + anchor. `certificate: pinned critical options must be present` and + `certificate: a bare name permits without requiring` disable the same line + in opposite directions — one drops the requirement, the other makes it + unconditional — and neither is redundant. Two mutations over one span + cannot be applied at once, so they go in different rounds. The first + version of this check applied everything in one pass and refused the whole + matrix on exactly that, calling a deliberate pair a collision. + + An anchor that vanishes *within* a round is still a refusal: that is an + overlap nobody declared, and two guards that quietly rewrite each other + cannot both be measured. + """ + # Guards sharing an anchor go in separate rounds; everything else rides + # along in the first. + rounds: list[list] = [] + seen: dict[tuple[str, str], int] = {} + for mutation in mutations: + _, path, old, _ = mutation + turn = seen.get((path, old), 0) + seen[(path, old)] = turn + 1 + while len(rounds) <= turn: + rounds.append([]) + rounds[turn].append(mutation) + + packages = [] + for crate in sorted({path.split("/")[0] for _, path, _, _ in mutations}): + packages += ["-p", _package_of(crate)] + + for round_number, batch in enumerate(rounds, start=1): + _check_one_round(batch, packages, in_flight, round_number, len(rounds)) + + +def _check_one_round(mutations, packages, in_flight, round_number, rounds): + print( + f" [{round_number}/{rounds}] compiling {len(mutations)} replacement(s)", + flush=True, + ) + touched: dict[Path, str] = {} + collided = [] + try: + for name, path, old, new in mutations: + source = REPO / path + if source not in touched: + touched[source] = source.read_text() + in_flight[str(source)] = touched[source] + current = source.read_text() + if old not in current: + collided.append((name, path)) + continue + source.write_text(current.replace(old, new, 1)) + + if collided: + lines = "\n".join(f" {name}\n in {path}" for name, path in collided) + raise SystemExit( + f"{len(collided)} anchor(s) vanished once their neighbours in " + f"the same round were applied, so those guards overlap without " + f"saying so and cannot both be measured:\n{lines}" + ) + + built = run(["cargo", "check", "--all-targets", *packages]) + if built.returncode != 0: + errors = [ + line for line in built.stderr.splitlines() if line.startswith("error") + ] + raise SystemExit( + f"a replacement does not compile (round {round_number} of " + f"{rounds}), so the guard it belongs to cannot be " + f"measured:\n" + + "\n".join(f" {line}" for line in errors[:10]) + + "\n\nRepoint it at something that builds before trusting any " + "number from this script." + ) + finally: + for source, original in touched.items(): + source.write_text(original) + in_flight.pop(str(source), None) + + +def write_artifact(*, partial: bool, results: list, refused: str | None, mode: str = "full"): + """The run's own record, so a claim about coverage can be checked later. + + Required by protocol amendment A2. It carries the guards, their named + discriminators, the per-guard verdict, and — the part that was missing — the + reason when the run refused to produce a number at all. A refusal used to + leave nothing behind, so "the matrix says 40/40" and "the matrix refused to + run" were indistinguishable a day later. + """ + (REPO / "tests" / "mutation-matrix.json").write_text( + json.dumps( + { + "generated": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "head": run(["git", "rev-parse", "HEAD"]).stdout.strip(), + "mode": mode, + "partial": partial, + "guards_total": len(MUTATIONS), + "guards_with_a_named_discriminator": len(DISCRIMINATES), + "refused": refused, + "results": results, + }, + indent=2, + ) + ) + + +def _python_test_names() -> set[str]: + names: set[str] = set() + print(" collecting the Python suites", flush=True) + collected = subprocess.run( + ["poetry", "run", "pytest", *SUITES, "--collect-only", "-q"], + cwd=REPO / "tests", + capture_output=True, + text=True, + ) + for line in collected.stdout.splitlines(): + if "::" in line: + names.add(line.split("::")[-1].split("[")[0].strip()) + return names + + +def _rust_test_names(crates) -> set[str]: + names: set[str] = set() + crates = list(crates) + for index, crate in enumerate(crates, start=1): + # One line per crate, because each of these builds that crate's test + # binary and the six together run for half an hour. The phase used to + # announce itself once and then go quiet for all of it, which is the + # same fault as the sweep's, one level down. + print(f" [{index}/{len(crates)}] listing tests in {crate}", flush=True) + listed = run(["cargo", "test", "-p", crate, "--", "--list"]) + for line in listed.stdout.splitlines(): + if line.endswith(": test"): + names.add(line.rsplit(":", 1)[0].split("::")[-1].strip()) + return names + + +def existing_tests(mutations=None) -> set[str]: + """Every test name this repository actually has, Python and Rust. + + Collected rather than assumed. `DISCRIMINATES` is a hand-written list of + names, and nothing checked that any of them existed — so an entry could + claim a guard was pinned by a test that had never been written. One was: + `test_a_certificate_with_a_different_key_id_is_refused` appeared nowhere in + the repository while the matrix reported on that guard for a week. + + An instrument built because we do not trust our tests was taking its own + list of test names on faith. + + Scoped to what is being checked, when a selection is given. Collecting + everything costs a full pytest collection plus one `cargo test --list` per + crate — eight of them, each building that crate's test binary — and that + fixed cost was paid identically for one guard and for forty-seven. A + reviewer ran a single guard and waited twenty-five minutes before the first + A/B, which is most of why the instrument reads as too expensive to run. + + The guarantee is not weakened: a discriminator missing from the narrow set + is looked for across the whole repository before it is called missing, so + the answer never depends on the guess that a guard's test lives in the same + crate as the guard. + """ + if mutations is None: + return _python_test_names() | _rust_test_names(RUST_CRATES) + + wanted = {t for name, *_ in mutations for t in DISCRIMINATES.get(name, [])} + # A leading `test_` is pytest's own convention and is what `run_named_only` + # already uses to route a name to one runner or the other. + want_python = any(t.startswith("test_") for t in wanted) + near = {path.split("/")[0] for _, path, _, _ in mutations} & set(RUST_CRATES) + + names = _rust_test_names(sorted(near)) + if want_python: + names |= _python_test_names() + if wanted - names: + names |= _rust_test_names(sorted(set(RUST_CRATES) - near)) + if not want_python: + names |= _python_test_names() + return names + + +def check_no_duplicate_entries(): + """`DISCRIMINATES` is a dict literal, and a repeated key wins silently. + + Not hypothetical: adding an entry for a guard that already had one produced + two keys spelled identically, naming two different tests, and Python kept + the second. The first entry simply stopped existing — no error, no warning, + and the count of covered guards did not move, which is the only reason it + was noticed. + + Reads this file's own source, because by the time the dict is built the + evidence is gone. + """ + tree = ast.parse(pathlib.Path(__file__).read_text()) + for node in tree.body: + if not isinstance(node, ast.Assign): + continue + target = node.targets[0] + if not (isinstance(target, ast.Name) and target.id == "DISCRIMINATES"): + continue + keys = [k.value for k in node.value.keys if isinstance(k, ast.Constant)] + repeated = sorted({k for k in keys if keys.count(k) > 1}) + if repeated: + listed = "\n".join(f" {k}" for k in repeated) + raise SystemExit( + f"{len(repeated)} guard(s) have more than one entry in " + f"DISCRIMINATES, and only the last one counts:\n{listed}" + ) + + names = [m[0] for m in MUTATIONS] + repeated = sorted({n for n in names if names.count(n) > 1}) + if repeated: + listed = "\n".join(f" {n}" for n in repeated) + raise SystemExit(f"{len(repeated)} guard name(s) are used twice:\n{listed}") + + +def check_discriminators(mutations): + """Every named discriminator must resolve to a test that exists. + + Added by protocol amendment A2. The check runs before anything is mutated, + for the same reason `check_anchors` does: a name that resolves to nothing + cannot fail, so a guard listing one can never be reported as caught, and the + run would spend an hour arriving at that. + """ + have = existing_tests(mutations) + if not have: + raise SystemExit( + "could not collect any test names, so the discriminator check " + "cannot run. Refusing rather than skipping it." + ) + + missing = [ + (name, test) + for name, *_ in mutations + for test in DISCRIMINATES.get(name, []) + if test not in have + ] + if missing: + lines = "\n".join(f" {test}\n named by {name}" for name, test in missing) + raise SystemExit( + f"{len(missing)} named discriminator(s) do not exist:\n{lines}\n\n" + "Write the test or repoint the entry. A name that resolves to " + "nothing is not weaker evidence than a real test — it is none." + ) + + +def failing_rust_tests(crate: str) -> set[str]: + """Which Rust tests fail, by name. + + Named rather than counted, so a guard pinned by a unit test can be credited + to that test rather than to "the Rust suite went red", which is the same + undiscriminating signal this script exists to stop trusting. + """ + result = run(["cargo", "test", "-p", crate]) + return { + line.split()[1].split("::")[-1] + for line in result.stdout.splitlines() + if line.startswith("test ") and line.rstrip().endswith("FAILED") + } + + +def restore_everything(): + """Put every file back, whatever happened. + + `run_one`'s `finally` covers a normal return and an exception. It does not + cover the process being killed, and this script runs for over an hour, so + being killed is a normal way for it to end. A guard left disabled in the + working tree then looks exactly like source code — and a `git add -A` at the + wrong moment commits it. That happened: the redirect policy that stops + `X-Vault-Token` following a 307 to another host was mutated away and landed + in a commit. + """ + for path, text in list(IN_FLIGHT.items()): + pathlib.Path(path).write_text(text) + IN_FLIGHT.pop(path, None) + LOCK.unlink(missing_ok=True) + + +def run_one(name, path, old, new): + """Disable one guard, run the suites, put the file back.""" + source = REPO / path + original = source.read_text() + IN_FLIGHT[str(source)] = original + source.write_text(original.replace(old, new, 1)) + try: + build = run(["cargo", "build", "--bin", "warpgate"]) + if build.returncode != 0: + return {"guard": name, "status": "did not compile"} + + started = time.time() + caught_by, tail = failing_tests() + elapsed = time.time() - started + + # Both crates, by name. Only `warpgate-vault` used to be consulted, and + # only as a yes/no — so the discriminating unit tests in + # `warpgate-protocol-ssh` were invisible to this script, and a Rust + # failure could not be attributed to a test. + for crate in RUST_CRATES: + caught_by |= failing_rust_tests(crate) + + expected = DISCRIMINATES.get(name) + if not caught_by: + status = "SURVIVED" + elif expected is None: + status = "no discriminating test named" + elif all(test in caught_by for test in expected): + # `all`, not `any` — protocol amendment A2. With `any`, an entry + # naming a Rust unit test and a Python integration test was + # satisfied by the unit test alone, and the integration test's + # discrimination was never established while the entry implied it + # had been. Every test an entry names now has to notice. + status = "caught" + else: + # Something failed, but not the test whose name claims this guard. + # The suite runs against a real sshd that enforces most of these + # itself, so an unrelated failure looks exactly like coverage. + status = "caught by something else" + + return { + "guard": name, + "status": status, + "expected": expected, + "caught_by": sorted(caught_by), + "seconds": round(elapsed), + "tail": "" if caught_by else tail.strip()[-200:], + } + finally: + source.write_text(original) + IN_FLIGHT.pop(str(source), None) + + +def main(): + # Refused rather than joined. The lock was written unconditionally, so a + # second run started happily beside a first and both rewrote the same source + # files — two mutations live at once, and every verdict either produces is + # about a tree neither of them describes. That happened, and it was noticed + # only by counting processes. + # + # A lock left by a killed run blocks this too, which is the right way round: + # a stale lock costs one command to clear, and a contaminated sweep costs a + # number that looks like evidence. + if LOCK.exists(): + sys.exit( + f"{LOCK} exists, so a run is in progress or one was killed before it " + "could clean up.\n" + "Two runs at once rewrite the same files and neither result means " + "anything.\n\n" + "If nothing is running — check with `pgrep -f mutation_matrix` — then " + "confirm no source was left mutated before clearing it:\n" + " python3 -c \"import sys; sys.path.insert(0,'.'); " + "import tests.mutation_matrix as m; " + "m.check_anchors(m.MUTATIONS + [m.CANARY])\"\n" + f" rm {LOCK}" + ) + LOCK.write_text("mutation_matrix is rewriting source files in place\n") + atexit.register(restore_everything) + for sig in (signal.SIGINT, signal.SIGTERM, signal.SIGHUP): + signal.signal(sig, lambda *_: sys.exit(130)) + + args = sys.argv[1:] + named_mode = "--named" in args + args = [a for a in args if a != "--named"] + + fail_fast = "--fail-fast" in args + args = [a for a in args if a != "--fail-fast"] + + changed_base = None + if "--changed" in args: + at = args.index("--changed") + if at + 1 >= len(args): + sys.exit("--changed needs a base revision to diff against") + changed_base = args[at + 1] + del args[at : at + 2] + + only = args[0] if args else "" + + # Skipped in `--named` mode, on Antigravity's ruling (amendment-signed + # 2026-08-18). This runs the whole `warpgate-vault` suite — 250 seconds, + # measured — on every invocation, to establish that the tree passes before + # anything is mutated. `--named` establishes the same thing per guard and + # more narrowly: the named test must pass *before* the mutation, or the + # guard is reported `already failing` and no verdict is produced. The + # precondition is redundant there and load bearing in the other mode. + if not named_mode: + rust_unit = run(["cargo", "test", "-p", "warpgate-vault", "-q"]) + if rust_unit.returncode != 0: + sys.exit("the tree does not pass before mutating; fix that first") + + if changed_base is not None: + diff = run(["git", "diff", "--name-only", f"{changed_base}...HEAD"]) + if diff.returncode != 0: + sys.exit( + f"could not diff against {changed_base!r}. In CI this usually " + "means the checkout has no history — fetch-depth: 0." + ) + touched = {line.strip() for line in diff.stdout.splitlines() if line.strip()} + selected = [m for m in MUTATIONS if m[1] in touched] + print( + f"{len(touched)} file(s) changed against {changed_base}; " + f"{len(selected)} of {len(MUTATIONS)} guards live in them" + ) + # Said out loud rather than left to be inferred from a green check: this + # mode cannot see a guard broken in a file the change did not touch. + # That is what the full sweep exists for, and a run that reports on a + # subset has to name the subset. + if not selected: + print("no guard's anchor file was touched; nothing to measure") + raise SystemExit(0) + else: + selected = [m for m in MUTATIONS if not only or only in m[0]] + if not selected: + sys.exit(f"no guard matches {only!r}") + # Each phase says it is starting. Together these can run for over an hour on + # a full sweep — the discriminator check builds a test binary per crate, and + # the replacement check runs two workspace-wide `cargo check` passes — and + # they used to do it in complete silence. A run that cannot be told apart + # from a hung one gets killed; that happened three times in one day. + print(f"checking {len(selected)} guard(s) before measuring anything", flush=True) + print(" anchors...", flush=True) + check_anchors(selected + [CANARY]) + check_no_duplicate_entries() + print(" named discriminators exist (builds a test binary per crate)...", flush=True) + check_discriminators(selected) + print(" replacements compile (two workspace passes)...", flush=True) + check_replacements_build(selected + [CANARY], IN_FLIGHT) + print(" ready\n", flush=True) + + if named_mode: + # Least-established first. A guard added yesterday is likelier to be + # wrong than one that has survived a dozen sweeps, and with --fail-fast + # that is the difference between learning in minutes and learning in + # hours. Order changes nothing about the verdicts: each guard is + # measured against its own baseline. + selected = sorted(selected, key=lambda m: m[0] not in RECENT_GUARDS) + results = verify_named(selected, fail_fast=fail_fast) + write_artifact(partial=bool(only), results=results, refused=None, mode="named") + good = [r for r in results if r["status"] == "discriminates"] + print( + f"\n{len(good)}/{len(results)} guards discriminate: the test named " + f"after the guard fails when the guard is disabled, and passes when " + f"it is not" + ) + # Anything short of all of them fails, including `no discriminating test + # named`. A guard nothing is pinned to is not a lesser pass — amendment + # A2 exists because such a guard was reported on for a week while never + # being disabled once. + for r in results: + if r["status"] != "discriminates": + print(f" {r['status']}: {r['guard']}") + raise SystemExit(0 if len(good) == len(results) else 1) + + # Before measuring anything, measure the instrument. + canary = run_one(*CANARY) + if canary["status"] != "SURVIVED": + refused = ( + "the canary was reported as caught, which cannot be true: it only " + "changes the text of a log line.\n" + f"Tests that failed: {', '.join(canary.get('caught_by', []))}\n" + "The suite is failing for reasons unrelated to the guards, so no " + "verdict from this run means anything. Fix that first." + ) + # A refusal is a result and gets written out like one. Amendment A2: + # every coverage number this project published was checkable only by + # repeating the run, and two of them were wrong. + write_artifact(partial=bool(only), results=[], refused=refused) + sys.exit(refused) + print(f"{'ok':>9} canary survived, so a 'caught' verdict means something\n") + + results = [] + for mutation in selected: + result = run_one(*mutation) + results.append(result) + mark = "ok" if result["status"] == "caught" else result["status"] + caught = len(result.get("caught_by", [])) + print(f"{mark:>9} {result['guard']} ({caught} tests)") + if result.get("tail"): + print(f" last output: {result['tail']}") + + run(["cargo", "build", "--bin", "warpgate"]) + write_artifact(partial=bool(only), results=results, refused=None) + + caught = [r for r in results if r["status"] == "caught"] + print( + f"\n{len(caught)}/{len(results)} guards are caught by the test named after them" + ) + for r in results: + if r["status"] != "caught": + print(f" {r['status']}: {r['guard']}") + if r["status"] == "caught by something else" and r["caught_by"]: + print(f" wanted {r['expected']}, got {r['caught_by'][:3]}") + if len(caught) != len(results): + # A partial result is the thing this script exists to stop us reporting + # as a whole one. + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/tests/stalling_host_key_server.py b/tests/stalling_host_key_server.py new file mode 100644 index 000000000..ab1a29406 --- /dev/null +++ b/tests/stalling_host_key_server.py @@ -0,0 +1,125 @@ +"""A server that presents a host key and then stops talking. + +The narrow window this exists to cover: Warpgate pauses its handshake deadline +while a host key is being decided on, because that wait can legitimately be a +person reading a fingerprint. The first version of that pause never ended — the +deadline was reset to a year out and nothing re-armed it — so a target that +offered an unknown key and then went silent was bounded by nothing except the +inactivity timeout, which is five minutes by default and hours wherever an +operator has raised it for interactive use. + +Reproducing it needs the stall to land *between* the host key arriving and the +transport finishing, which is a couple of messages wide. So this server sends +everything up to and including the key exchange reply — the message that carries +the host key — and then sends nothing further, in particular no NEWKEYS. The +client has the key, answers the question, and then waits forever for a handshake +that will not complete. + +Stalling later, on authentication, would not test this: by then the transport is +done, `connect` has returned, and a different bound applies. +""" + +import socket +import threading + +import paramiko + +from .util import alloc_port + +# The key exchange replies that carry the server's host key. Everything up to +# one of these is sent normally; nothing after it is sent at all. +KEXDH_REPLY = 31 +KEX_ECDH_REPLY = 33 + + +class _MuteAfterHostKey(paramiko.Transport): + """Sends the host key, then drops every outbound message.""" + + def __init__(self, sock, delivered: threading.Event): + super().__init__(sock) + self._delivered = delivered + self._mute = False + + def _send_message(self, data): + if self._mute: + return + # `asbytes()` on a Message gives the packet with its type byte first. + raw = data.asbytes() if hasattr(data, "asbytes") else bytes(data) + kind = raw[0] if raw else 0 + super()._send_message(data) + if kind in (KEXDH_REPLY, KEX_ECDH_REPLY): + self._mute = True + self._delivered.set() + + +class _Server(paramiko.ServerInterface): + def get_allowed_auths(self, username): + return "password" + + def check_auth_password(self, username, password): + return paramiko.AUTH_FAILED + + +class StallingHostKeyServer: + """Listens on a port, offers a host key nobody has seen, then goes quiet.""" + + def __init__(self): + self.port = alloc_port() + self.key_delivered = threading.Event() + self.connections = 0 + self._stop = threading.Event() + self._sock = None + self._thread = None + + def start(self): + # Dual-stack, for the reason `hostile_ssh_server.py` gives: Warpgate + # dials `localhost` and takes the first address it resolves, which here + # is `::1`. A v4-only listener is never reached, and the test then + # passes for having tested nothing. + self._sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + self._sock.bind(("::", self.port)) + self._sock.listen(8) + self._sock.settimeout(0.5) + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + return self + + def _serve(self): + # Generated per instance, so it is a key Warpgate has never trusted and + # the unknown-host-key branch is the one that runs. + host_key = paramiko.RSAKey.generate(2048) + while not self._stop.is_set(): + try: + client, _ = self._sock.accept() + except OSError: + continue + self.connections += 1 + threading.Thread( + target=self._session, args=(client, host_key), daemon=True + ).start() + + def _session(self, client, host_key): + transport = None + try: + transport = _MuteAfterHostKey(client, self.key_delivered) + transport.add_server_key(host_key) + # Returns once the handshake stalls or fails; either way the socket + # is held open below so the client is the one that gives up. + transport.start_server(server=_Server()) + except Exception: + pass + finally: + self._stop.wait(300) + try: + if transport is not None: + transport.close() + client.close() + except OSError: + pass + + def stop(self): + self._stop.set() + if self._sock is not None: + self._sock.close() diff --git a/tests/stalling_jump_host.py b/tests/stalling_jump_host.py new file mode 100644 index 000000000..6cfd62a68 --- /dev/null +++ b/tests/stalling_jump_host.py @@ -0,0 +1,117 @@ +"""A jump host that authenticates you and then never opens the tunnel. + +The six modes in `hostile_ssh_server.py` all fail at or before the banner, which +is enough to exercise the outbound handshake deadline and nothing past it. This +one has to speak real SSH — complete key exchange, accept an authentication — +because the step under test comes after both: `channel_open_direct_tcpip`, the +request that asks a jump host to reach the next hop. + +That step had no bound of its own. Each hop's deadline is armed inside +`wait_for_connection`, which runs *after* the tunnel is open, so a jump host that +accepts the request and answers nothing stalled the connection for as long as the +previous hop's inactivity timeout allowed — five minutes by default, hours +wherever an operator has raised it for interactive sessions. + +Password auth deliberately: the point is the tunnel step, and making the jump +host negotiate an OpenSSH certificate would put paramiko's algorithm support in +the middle of a test about something else. +""" + +import socket +import threading + +import paramiko + +from .util import alloc_port + +PASSWORD = "let-me-through" + + +class _Server(paramiko.ServerInterface): + def __init__(self, opened: threading.Event): + self.opened = opened + + def get_allowed_auths(self, username): + return "password" + + def check_auth_password(self, username, password): + if password == PASSWORD: + return paramiko.AUTH_SUCCESSFUL + return paramiko.AUTH_FAILED + + def check_channel_request(self, kind, chanid): + return paramiko.OPEN_SUCCEEDED + + def check_channel_direct_tcpip_request(self, chanid, origin, destination): + """Accept the request and answer nothing. + + Paramiko calls this on the transport thread, so blocking here is exactly + the behaviour being simulated: the peer has the request and sends back + neither a confirmation nor a failure. Longer than the deadline under + test, so the client is the one that gives up. + """ + self.opened.set() + threading.Event().wait(300) + return paramiko.OPEN_FAILED_CONNECT_FAILED + + +class StallingJumpHost: + """Listens on a port, authenticates, and stalls on the tunnel request.""" + + def __init__(self): + self.port = alloc_port() + self.tunnel_requested = threading.Event() + self.connections = 0 + self._stop = threading.Event() + self._sock = None + self._thread = None + + def start(self): + # Dual-stack, for the reason `hostile_ssh_server.py` gives: Warpgate + # dials `localhost` and takes the first address it resolves, which here + # is `::1`. A v4-only listener is never reached at all, and the test then + # passes for having tested nothing — which is exactly what happened on + # the first run of this one. + self._sock = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) + self._sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._sock.setsockopt(socket.IPPROTO_IPV6, socket.IPV6_V6ONLY, 0) + self._sock.bind(("::", self.port)) + self._sock.listen(8) + self._sock.settimeout(0.5) + self._thread = threading.Thread(target=self._serve, daemon=True) + self._thread.start() + return self + + def _serve(self): + host_key = paramiko.RSAKey.generate(2048) + while not self._stop.is_set(): + try: + client, _ = self._sock.accept() + except OSError: + continue + self.connections += 1 + threading.Thread( + target=self._session, args=(client, host_key), daemon=True + ).start() + + def _session(self, client, host_key): + try: + transport = paramiko.Transport(client) + transport.add_server_key(host_key) + transport.start_server(server=_Server(self.tunnel_requested)) + # The stall happens on the transport thread inside + # `check_channel_direct_tcpip_request`; nothing to do here but hold + # the session open until the client gives up or the test ends. + self._stop.wait(300) + except Exception: + pass + finally: + try: + client.close() + except OSError: + pass + + def stop(self): + self._stop.set() + if self._sock is not None: + self._sock.close() diff --git a/tests/stub_vault.py b/tests/stub_vault.py new file mode 100644 index 000000000..99ea0ae83 --- /dev/null +++ b/tests/stub_vault.py @@ -0,0 +1,431 @@ +"""A stand-in for Vault's SSH secrets engine. + +Signs with a throwaway CA through ``ssh-keygen``, so the certificate tests need +neither a Vault server nor a cluster. Every knob here exists to reproduce a +failure Warpgate has to survive, and every recorded request exists so a test can +assert on what Warpgate actually asked for. +""" + +import json +import subprocess +import tempfile +import threading +import time +from base64 import b64decode, urlsafe_b64decode, urlsafe_b64encode +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from pathlib import Path +from urllib.parse import parse_qs, urlsplit + +MOUNT = "ssh-client-signer" + + +def jwt(claims: dict) -> str: + """A JWT-shaped token. Unsigned — the stub checks shape and claims, which is + what catches a payload built wrong, not cryptographic validity.""" + + def segment(raw: bytes) -> str: + return urlsafe_b64encode(raw).rstrip(b"=").decode() + + return ".".join( + [ + segment(json.dumps({"alg": "RS256", "typ": "JWT"}).encode()), + segment(json.dumps(claims).encode()), + segment(b"stub-signature"), + ] + ) + + +def jwt_claims(token: str) -> dict: + payload = token.split(".")[1] + return json.loads(urlsafe_b64decode(payload + "=" * (-len(payload) % 4))) + + +def is_jwt(token) -> bool: + parts = (token or "").split(".") + if len(parts) != 3 or not all(parts): + return False + try: + jwt_claims(token) + except Exception: + return False + return True + + +SERVICE_ACCOUNT_JWT = jwt( + {"iss": "kubernetes/serviceaccount", "sub": "system:serviceaccount:warpgate:warpgate"} +) + + +def reject_login(method, body): + """Checks what the real auth method checks, in the same order it does. + + A stub that accepts anything turns every login test into an assertion + about the stub instead of about the payload Warpgate built. + """ + if method == "kubernetes": + if not body.get("role"): + return "missing role" + if not is_jwt(body.get("jwt")): + return "jwt is not a JWT" + return None + + if method == "approle": + if not body.get("role_id") or not body.get("secret_id"): + return "missing role_id or secret_id" + return None + + if method == "aws": + return reject_aws_login(body) + + if method == "azure": + for field in ("subscription_id", "resource_group_name", "vm_name"): + if not body.get(field): + return f"missing {field}" + if "vmss_name" in body and not body["vmss_name"]: + return "vmss_name present but empty" + if not is_jwt(body.get("jwt")): + return "jwt is not a JWT" + return None + + if method == "gcp": + role = body.get("role") + if not role: + return "missing role" + if not is_jwt(body.get("jwt")): + return "jwt is not a JWT" + # Vault rejects a token minted for a different audience, which is + # what stops a token issued for one role being replayed at another. + if jwt_claims(body["jwt"]).get("aud") != f"vault/{role}": + return "jwt audience is not bound to the role" + return None + + return f"unknown auth method {method}" + +def reject_aws_login(body): + """Vault replays this request against STS, so it has to be a signed + GetCallerIdentity call and not merely four non-empty fields.""" + if body.get("iam_http_request_method") != "POST": + return "iam_http_request_method must be POST" + try: + url = b64decode(body.get("iam_request_url", "")).decode() + signed_body = b64decode(body.get("iam_request_body", "")).decode() + headers = json.loads(b64decode(body.get("iam_request_headers", ""))) + except Exception as e: + return f"undecodable IAM payload: {e}" + + host = urlsplit(url).hostname or "" + if host != "sts.amazonaws.com" and not ( + host.startswith("sts.") and host.endswith(".amazonaws.com") + ): + return f"not an STS endpoint: {host}" + if parse_qs(signed_body).get("Action") != ["GetCallerIdentity"]: + return f"not a GetCallerIdentity call: {signed_body}" + + authorization = next( + (v for k, v in headers.items() if k.lower() == "authorization"), "" + ) + if not authorization.startswith("AWS4-HMAC-SHA256"): + return "missing SigV4 Authorization header" + return None + + +class Recorder: + """Records every request it receives and nothing else. Used to prove that a + redirect from the issuer never reaches its target.""" + + def __init__(self): + self.requests = [] + handler = self + + class _RecordingHandler(BaseHTTPRequestHandler): + def do_GET(self): + self._record() + + def do_POST(self): + self._record() + + def _record(self): + handler.requests.append((self.path, dict(self.headers))) + self.send_response(200) + self.send_header("Content-Length", "0") + self.end_headers() + + def log_message(self, *args): + pass + + self._server = ThreadingHTTPServer(("127.0.0.1", 0), _RecordingHandler) + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + + def start(self): + self._thread.start() + + def stop(self): + self._server.shutdown() + self._thread.join(timeout=5) + self._server.server_close() + + @property + def url(self): + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + +class _Handler(BaseHTTPRequestHandler): + def do_GET(self): + """The cloud metadata services Warpgate reads its identity from.""" + stub = self.server.stub + path = self.path.split("?")[0] + stub.requests.append(self.path) + stub.metadata_requests.append(self.path) + + query = parse_qs(urlsplit(self.path).query) + + if path == "/metadata/identity/oauth2/token": + resource = query.get("resource", [""])[0] + self._reply(200, {"access_token": jwt({"aud": resource, "oid": "oid-1"})}) + elif path == "/metadata/instance/compute": + self._reply(200, { + "subscriptionId": "sub-1", + "resourceGroupName": "rg-1", + "name": "vm-1", + "vmScaleSetName": "", + }) + elif path.endswith("/service-accounts/default/identity"): + audience = query.get("audience", [""])[0] + self._send_text(200, jwt({"aud": audience, "sub": "gce-instance"}) + "\n") + else: + self._reply(404, {"errors": [f"no handler for {path}"]}) + + def _send_text(self, status, body): + encoded = body.encode() + self.send_response(status) + self.send_header("Content-Type", "text/plain") + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + self.wfile.write(encoded) + + def do_POST(self): + stub = self.server.stub + stub.requests.append(self.path) + length = int(self.headers.get("Content-Length", 0)) + body = json.loads(self.rfile.read(length) or b"{}") + + if self.path == "/v1/sys/wrapping/unwrap": + token = self.headers.get("X-Vault-Token", "") + if not token: + self._reply(400, {"errors": ["missing wrapping token"]}) + return + stub.unwraps.append(token) + # A wrapping token can be redeemed exactly once. Modelling that is + # the difference between a test that proves the secret ID survives + # more than one login and one that never asks. + if token in stub.spent_wrapping_tokens: + self._reply(400, {"errors": ["wrapping token is not valid or does not exist"]}) + return + stub.spent_wrapping_tokens.add(token) + self._reply(200, {"data": {"secret_id": "unwrapped-secret-id"}}) + return + + if self.path.startswith("/v1/auth/") and self.path.endswith("/login"): + method = self.path.split("/")[3] + rejection = reject_login(method, body) + if rejection: + self._reply(400, {"errors": [rejection]}) + return + + stub.logins.append({"method": method, **body}) + + # Vault is not instantaneous, and how many sessions can be inside a + # single login at once is the thing some tests are about. + time.sleep(stub.login_delay) + + stub.valid_token = f"stub-token-{len(stub.logins)}" + self._reply( + 200, + { + "auth": { + "client_token": stub.valid_token, + "lease_duration": stub.lease_duration, + } + }, + ) + return + + if self.path.startswith(f"/v1/{MOUNT}/sign/"): + self._sign(stub, self.path.rsplit("/", 1)[-1], body) + return + + self._reply(404, {"errors": [f"no handler for {self.path}"]}) + + + def _sign(self, stub, role, body): + presented = self.headers.get("X-Vault-Token") + stub.signs.append({"role": role, "token": presented, **body}) + + if stub.sign_redirect_to is not None: + self.send_response(307) + self.send_header("Location", stub.sign_redirect_to) + self.send_header("Content-Length", "0") + self.end_headers() + return + + if stub.sign_error_body is not None: + self._send_raw(500, stub.sign_error_body) + return + + if stub.sign_status is not None: + self._reply(stub.sign_status, {"errors": ["stub refuses to sign"]}) + return + + # Mirrors Vault rejecting a token that was revoked, or that predates a + # restart, before its lease was due to expire. Only checked once a test + # asks for it, so a token cached by an earlier test cannot leak into an + # unrelated one. + if not stub.accept_any_token and presented != stub.valid_token: + self._reply(403, {"errors": ["permission denied", "invalid token"]}) + return + + if stub.sign_data is not None: + self._reply(200, {"data": stub.sign_data}) + return + + if stub.signed_key is not None: + self._reply(200, {"data": {"signed_key": stub.signed_key}}) + return + + certificate = stub.issue( + public_key=stub.sign_public_key or body["public_key"], + principals=( + stub.principals if stub.principals is not None else body["valid_principals"] + ), + key_id=stub.sign_key_id if stub.sign_key_id is not None else body.get("key_id", ""), + ) + self._reply(200, {"data": {"signed_key": certificate}}) + + def _reply(self, status, payload): + self._send_raw(status, json.dumps(payload).encode(), "application/json") + + def _send_raw(self, status, encoded: bytes, content_type="application/json"): + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(encoded))) + self.end_headers() + try: + self.wfile.write(encoded) + except (BrokenPipeError, ConnectionResetError): + # Warpgate hangs up on a body larger than it will accept, which is + # exactly what one of the tests here is checking for. + pass + + def log_message(self, *args): + pass + + +class StubVault: + def __init__(self, directory: Path): + directory.mkdir(parents=True, exist_ok=True) + self.directory = directory + self.ca_key = directory / "ca" + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-f", str(self.ca_key), "-N", ""], + check=True, + ) + self._server = ThreadingHTTPServer(("127.0.0.1", 0), _Handler) + self._server.stub = self + self._thread = threading.Thread(target=self._server.serve_forever, daemon=True) + self.logins = [] + self.signs = [] + self.requests = [] + self.metadata_requests = [] + self.unwraps = [] + self.spent_wrapping_tokens = set() + self.valid_token = None + self.reset() + + def reset(self): + """Restores the defaults a happy-path test expects.""" + self.lease_duration = 3600 + self.login_delay = 0 + self.accept_any_token = True + self.sign_status = None + self.sign_error_body = None + self.sign_redirect_to = None + self.signed_key = None + self.sign_data = None + self.sign_public_key = None + self.principals = None + self.validity = "-30s:+2m" + self.cert_type = "user" + self.sign_options = [] + self.sign_key_id = None + self.logins.clear() + self.signs.clear() + self.requests.clear() + self.metadata_requests.clear() + self.unwraps.clear() + self.spent_wrapping_tokens.clear() + + def start(self): + self._thread.start() + + def stop(self): + self._server.shutdown() + self._thread.join(timeout=5) + self._server.server_close() + + @property + def url(self): + host, port = self._server.server_address[:2] + return f"http://{host}:{port}" + + @property + def ca_public_key(self) -> str: + return Path(f"{self.ca_key}.pub").read_text().strip() + + def issue(self, public_key: str, principals: str, key_id: str) -> str: + with tempfile.TemporaryDirectory() as directory: + key = Path(directory) / "key.pub" + key.write_text(public_key) + # ssh-keygen decides itself which of `sign_options` are critical + # options and which are extensions, exactly as a Vault role's + # default_critical_options and default_extensions end up doing. + # + # `clear` first, then only what the test asked for. Left to itself + # ssh-keygen grants all five standard extensions — including + # permit-port-forwarding and permit-agent-forwarding — which no + # sensible Vault role does: the usual `default_extensions` is + # `{"permit-pty": ""}`. The stub was quietly issuing more privilege + # than the thing it stands in for, so a target with the default + # extension allow-list would have refused every certificate in this + # suite, and any test about forwarding would have been meaningless. + options = ["-O", "clear", "-O", "permit-pty"] + options += [arg for option in self.sign_options for arg in ("-O", option)] + subprocess.run( + [ + "ssh-keygen", "-q", + "-s", str(self.ca_key), + "-I", key_id, + "-n", principals, + "-V", self.validity, + *(["-h"] if self.cert_type == "host" else []), + *options, + str(key), + ], + check=True, + ) + return (Path(directory) / "key-cert.pub").read_text() + + def unrelated_public_key(self) -> str: + """A public key Warpgate does not hold the private half of.""" + path = self.directory / "unrelated" + if not path.exists(): + subprocess.run( + ["ssh-keygen", "-q", "-t", "ed25519", "-f", str(path), "-N", ""], + check=True, + ) + return Path(f"{path}.pub").read_text().strip() + + def invalidate_token(self): + """Makes the cached token stop working, as a Vault restart would.""" + self.accept_any_token = False + self.valid_token = "rotated-away" diff --git a/tests/test_api.py b/tests/test_api.py index 6b682a7f8..02e16244a 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -4,6 +4,7 @@ """ import contextlib +import tempfile from dataclasses import dataclass from typing import Callable, Dict, Optional, Set from json import load @@ -61,6 +62,11 @@ def make_limited_admin_role_payload(**overrides): } +# A fresh directory per run rather than a fixed name in a world-writable one. +# `/tmp/recordings-test` can be pre-created as a symlink by any other user on a +# shared machine, and this case asks Warpgate to write there. +RECORDINGS_TEST_PATH = tempfile.mkdtemp(prefix="warpgate-recordings-") + ADMIN_API_TEST_CASES: list[AdminApiTestCase] = [ AdminApiTestCase( id="get_sessions", @@ -719,7 +725,7 @@ def make_limited_admin_role_payload(**overrides): call=lambda api, r: api.test_recordings_storage_with_http_info( sdk.RecordingsStorageConfig( sdk.RecordingsStorageConfigRecordingsDiskConfig( - kind="Disk", path="/tmp/recordings-test" + kind="Disk", path=RECORDINGS_TEST_PATH ) ) ), diff --git a/tests/test_ssh_target_cert_auth.py b/tests/test_ssh_target_cert_auth.py new file mode 100644 index 000000000..1aa64f54e --- /dev/null +++ b/tests/test_ssh_target_cert_auth.py @@ -0,0 +1,1596 @@ +"""Target authentication with short-lived certificates issued by Vault. + +Beyond the happy path, the cases here are drawn from two places: the failures +this feature hit while being built, and the classes of bug Warpgate has actually +shipped before — a credential accepted without proof of possession +(GHSA-3cjp-w4cp-m9c8), a new code path skipping a control every other path +applies (GHSA-qmr2-wp96-h9ff), and an identity taken from the wrong stage of +authentication (GHSA-c94j-vqr5-3mxr). +""" + +import json +import time +from pathlib import Path +from uuid import uuid4 + +import psutil +import pytest +import yaml + +from .api_client import admin_client, sdk +from .conftest import TARGET_HOST, ProcessManager, WarpgateProcess +from .stub_vault import ( + SERVICE_ACCOUNT_JWT, + Recorder, + StubVault, + jwt_claims, + reject_login, +) +from .util import wait_port + +USER_PUBLIC_KEY_PATH = Path("ssh-keys/id_ed25519.pub") +USER_PRIVATE_KEY_PATH = "ssh-keys/id_ed25519" + + +@pytest.fixture(scope="module") +def stub_vault(ctx): + stub = StubVault(ctx.tmpdir / f"stub-vault-{uuid4()}") + stub.start() + yield stub + stub.stop() + + +@pytest.fixture(scope="module") +def cert_wg(processes: ProcessManager, ctx, stub_vault: StubVault): + """Warpgate wired to the stub issuer, with its log kept for assertions.""" + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + log_path = ctx.tmpdir / f"cert-wg-{uuid4()}.log" + with log_path.open("w") as log: + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + }, + stdout=log, + stderr=log, + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + wg.log_path = log_path + yield wg + + +@pytest.fixture(scope="module") +def cert_ssh_port(processes: ProcessManager, stub_vault: StubVault): + """A target that trusts the stub CA and has no authorized_keys at all.""" + port = processes.start_ssh_server(trusted_ca=[stub_vault.ca_public_key]) + wait_port(port) + return port + + +@pytest.fixture(autouse=True) +def reset_stub(stub_vault: StubVault): + stub_vault.reset() + yield + stub_vault.reset() + + +def make_user_and_target( + api, + ssh_port, + *, + role=None, + username="root", + assign=True, + allowed_critical_options=None, +): + wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user(sdk.CreateUserRequest(username=f"user-{uuid4()}")) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, wg_role.id) + + target = api.create_target( + sdk.TargetDataRequest( + name=f"cert-{uuid4()}", + options=sdk.TargetOptions( + sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host=TARGET_HOST, + port=ssh_port, + username=username, + auth=sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetCertificateAuth( + kind="Certificate", + role=role, + allowed_critical_options=[ + sdk.SshCertificateCriticalOption(name=name, value=value) + for name, value in (allowed_critical_options or []) + ], + ) + ), + ) + ), + ) + ) + if assign: + api.add_target_role(target.id, wg_role.id) + return user, target + + +def start(processes: ProcessManager, wg: WarpgateProcess, user, target, *extra): + return processes.start_ssh_client( + f"{user.username}:{target.name}@localhost", + "-p", + str(wg.ssh_port), + "-o", + f"IdentityFile={USER_PRIVATE_KEY_PATH}", + "-o", + "PreferredAuthentications=publickey", + *extra, + "ls", + "/bin/sh", + ) + + +def connect(processes: ProcessManager, wg: WarpgateProcess, user, target, timeout, *extra): + client = start(processes, wg, user, target, *extra) + stdout = client.communicate(timeout=timeout)[0] + return client.returncode, stdout + + +def log_since(wg: WarpgateProcess, offset: int) -> str: + """Only what this test wrote. The gateway is shared by the whole module, so + searching the whole log would find another test's line just as happily.""" + return Path(wg.log_path).read_text(errors="replace")[offset:] + + +@pytest.fixture +def api(cert_wg: WarpgateProcess): + with admin_client(f"https://localhost:{cert_wg.http_port}") as client: + yield client + + +class TestHappyPath: + def test_connects_with_an_issued_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert code == 0 + assert stdout == b"/bin/sh\n" + assert len(stub_vault.signs) == 1 + assert stub_vault.signs[0]["valid_principals"] == "root" + assert stub_vault.signs[0]["cert_type"] == "user" + + def test_uses_the_default_role_unless_the_target_names_one( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + assert stub_vault.signs[-1]["role"] == "warpgate" + + user, target = make_user_and_target(api, cert_ssh_port, role="privileged") + connect(processes, cert_wg, user, target, timeout) + assert stub_vault.signs[-1]["role"] == "privileged" + + def test_each_session_gets_a_fresh_key_and_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + for _ in range(3): + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + offered = [sign["public_key"] for sign in stub_vault.signs] + assert len(offered) == 3 + assert len(set(offered)) == 3, "an ephemeral key was reused between sessions" + + def test_the_token_is_reused_across_sessions( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + for _ in range(3): + connect(processes, cert_wg, user, target, timeout) + + # The first session may find a token cached by an earlier test, so the + # assertion is that three sessions do not each cause a login. + assert len(stub_vault.logins) <= 1 + + def test_no_ttl_is_requested_unless_one_is_configured( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Vault reads an absent `ttl` as the role's default. Sending a zero + instead would ask for the shortest certificate Vault will issue.""" + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + assert "ttl" not in stub_vault.signs[-1] + + def test_a_configured_ttl_is_asked_for( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """The TTL can be held down from Warpgate's side without editing the + Vault role — Vault clamps it to the role's `max_ttl` regardless.""" + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "certificate_ttl": "90s", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + assert stub_vault.signs[-1]["ttl"] == "90s" + + +class TestIdentity: + """The certificate must name the user Warpgate actually authenticated.""" + + def test_key_id_carries_the_authenticated_user_and_session( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + key_id = stub_vault.signs[-1]["key_id"] + prefix, username, session = key_id.split(":") + assert prefix == "warpgate" + assert username == user.username + assert session + + def test_the_target_username_is_not_the_warpgate_username( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """`valid_principals` bounds access on the target and must never be + taken from the identity the client chose to log into Warpgate with.""" + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + assert stub_vault.signs[-1]["valid_principals"] == "root" + assert user.username not in stub_vault.signs[-1]["valid_principals"] + + def test_a_comma_in_the_target_username_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Vault reads `valid_principals` as a comma-separated list, so a comma + in the target's username would ask for a certificate good for accounts + nobody named. No request may leave at all.""" + user, target = make_user_and_target(api, cert_ssh_port, username="deploy,root") + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + + assert stub_vault.signs == [] + + def test_a_newline_in_the_target_username_never_reaches_vault( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The principal is written into the certificate and the certificate is + written into the target's sshd log, so a newline in a target's username + is a way to compose log lines on the target. Nothing may be sent at all + — not even the login that would precede it.""" + user, target = make_user_and_target(api, cert_ssh_port, username="root\nnobody") + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + + assert stub_vault.requests == [] + + def test_a_ticket_session_is_named_after_its_user_and_not_its_secret( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The key ID must name the user Warpgate authenticated, however it + authenticated them. A ticket session logs in as `ticket-`, so a + key ID taken from the login name rather than from the authentication + result would name nobody and copy the ticket secret into the target's + sshd log — the shape of GHSA-c94j-vqr5-3mxr.""" + user, target = make_user_and_target(api, cert_ssh_port) + secret = api.create_ticket( + sdk.CreateTicketRequest(target_name=target.name, username=user.username) + ).secret + + client = processes.start_ssh_client( + f"ticket-{secret}@localhost", + "-p", + str(cert_wg.ssh_port), + "-o", + "PreferredAuthentications=password", + "-i", + "/dev/null", + "ls", + "/bin/sh", + password="irrelevant", + ) + assert client.communicate(timeout=timeout)[0] == b"/bin/sh\n" + + key_id = stub_vault.signs[-1]["key_id"] + assert key_id.startswith(f"warpgate:{user.username}:") + assert secret not in key_id + + +class TestRejections: + def test_target_that_does_not_trust_the_ca( + self, processes, cert_wg, stub_vault, api, timeout + ): + port = processes.start_ssh_server() + wait_port(port) + + user, target = make_user_and_target(api, port) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert code != 0 + assert stdout == b"" + assert len(stub_vault.signs) == 1, "the target rejected it, not Warpgate" + + def test_expired_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Warpgate refuses it before offering it, and says so. + + A non-zero exit code proves nothing here: the target's own sshd refuses + an expired certificate whether or not Warpgate looked. Deleting the + expiry check would leave this test passing for the target's reasons, so + it asserts on the one message only that check produces. + """ + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.validity = "-2h:-1h" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + assert "already expired" in log_since(cert_wg, offset) + + def test_certificate_that_is_not_yet_valid( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A target whose clock lags far enough behind Warpgate's sees this. + + Warpgate has no `valid_after` check of its own — the refusal is entirely + the target's — so what is asserted is Warpgate's contribution: the + window it reports and the hint naming the clock. Without that, whoever + is debugging goes looking at credentials that are fine. + """ + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.validity = "+1h:+2h" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + assert "check the target's clock" in log_since(cert_wg, offset) + + def test_certificate_for_a_different_principal( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The target would refuse an account it does not have, so a failed + connection says nothing about the principal check. This asserts the + refusal Warpgate itself produces, before anything is offered.""" + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.principals = "nobody" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + assert "rather than only the target account" in log_since(cert_wg, offset) + + def test_certificate_issued_for_a_key_warpgate_does_not_hold( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A certificate is a public credential. Signing someone else's key must + not authenticate anyone — the class of bug behind GHSA-3cjp-w4cp-m9c8, + where an SSH key offer was accepted without a signature.""" + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.sign_public_key = stub_vault.unrelated_public_key() + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + # The target would refuse it too, but then the log would say only that + # the target said no. Warpgate knows which key it generated. + assert "signed a key other than" in log_since(cert_wg, offset) + + def test_a_host_certificate_is_not_offered_to_the_target( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A host certificate can never authenticate a user, so an issuer that + returns one is misconfigured or lying. Either way the reason belongs in + Warpgate's log rather than arriving as an unexplained rejection.""" + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.cert_type = "host" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + assert "host certificate" in log_since(cert_wg, offset) + + def test_malformed_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.signed_key = "this is not a certificate" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + + @pytest.mark.parametrize( + "data", + [ + pytest.param({}, id="no-signed-key"), + pytest.param({"signed_key": ""}, id="empty-signed-key"), + pytest.param({"signed_key": 42}, id="signed-key-is-not-a-string"), + ], + ) + def test_a_success_that_carries_no_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout, data + ): + """`200 OK` is not a certificate. Each of these parses as JSON and none + of them is a credential, so the session has to end rather than continue + with whatever an absent field defaults to.""" + stub_vault.sign_data = data + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert len(stub_vault.signs) == 1 + + def test_an_unexpected_forced_command_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The target's sshd enforces critical options, so an issuer that + attaches `force-command` decides what the session runs — under the + user's own principal and key ID, which makes the target's log attribute + the issuer's command to them. Planting one needs only write access to a + Vault role, not the right to sign or a route to the target, so Warpgate + is the only place this can be caught.""" + stub_vault.sign_options = ["force-command=echo chosen-by-the-issuer"] + user, target = make_user_and_target(api, cert_ssh_port) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert code != 0 + assert b"chosen-by-the-issuer" not in stdout, "the forced command ran" + assert stub_vault.signs, "no certificate was issued, so nothing was refused" + + def test_the_refusal_reaches_the_user_not_only_the_log( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A server-side warning nobody is watching is not a control. Whoever is + connecting has to be told, and told that it was Warpgate that refused — + "the target rejected you" sends them to the wrong machine.""" + stub_vault.sign_options = ["force-command=echo chosen-by-the-issuer"] + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + stdout = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert "Warpgate refused the certificate" in stdout + assert "force-command" in stdout + + def test_a_critical_option_the_target_expects_is_allowed_through( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A restricted Vault role may set `default_critical_options` on + purpose. Naming it on the target is how an operator says so.""" + stub_vault.sign_options = ["force-command=echo expected-by-the-operator"] + user, target = make_user_and_target( + api, + cert_ssh_port, + allowed_critical_options=[ + ("force-command", "echo expected-by-the-operator") + ], + ) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert (code, stdout) == (0, b"expected-by-the-operator\n") + + def test_a_pinned_value_must_match( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Allowing the name alone would let the issuer choose the command; the + point of pinning is that the value is the part that matters.""" + stub_vault.sign_options = ["force-command=echo something-else-entirely"] + user, target = make_user_and_target( + api, + cert_ssh_port, + allowed_critical_options=[("force-command", "echo the-expected-one")], + ) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert code != 0 + assert b"something-else-entirely" not in stdout + + def test_a_certificate_naming_the_wrong_account_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Vault returns the requested principals verbatim or refuses, so a set + that omits the account being reached did not come from this request. + + The target's sshd would refuse this certificate too, which is exactly + why the assertion is on who did the refusing: without Warpgate's own + check the session still fails, just with the wrong explanation and + after the certificate has been put on the wire. + """ + stub_vault.principals = "someone-else" + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + stdout = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert client.returncode != 0 + assert stub_vault.signs, "no certificate was issued, so nothing was refused" + assert "Warpgate refused the certificate" in stdout + assert "rather than only the target account root" in stdout + + +class TestIssuerFailures: + def test_issuer_refuses_to_sign( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.sign_status = 403 + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + # Without this the test passes just as well when the certificate + # path never ran at all. + assert stub_vault.signs, "no certificate was ever requested" + + def test_a_persistent_denial_is_not_retried_forever( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """One re-login distinguishes a stale token from a policy denial. A + second failure has to be final, or a denied target becomes a request + amplifier against Vault.""" + stub_vault.sign_status = 403 + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + assert len(stub_vault.signs) == 2 + + def test_a_stale_token_is_refreshed_once( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Vault can reject a token long before its lease runs out — after a + restart, or a revocation. Warpgate must recover within the session + rather than failing until the lease expires.""" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + stub_vault.invalidate_token() + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + assert len(stub_vault.signs) == 3, "expected one rejected and one retried sign" + + def test_a_redirect_from_the_issuer_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Following a redirect would hand `X-Vault-Token` to whatever host it + names — reqwest only knows to strip `Authorization`. The session must + fail instead, and the redirect target must never be contacted.""" + recorder = Recorder() + recorder.start() + try: + stub_vault.sign_redirect_to = f"{recorder.url}/v1/steal" + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + assert stub_vault.signs, "the signing request was never made" + assert recorder.requests == [], "the redirect was followed" + finally: + recorder.stop() + + def test_an_oversized_error_body_is_not_relayed_to_the_client( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """An endpoint answering on the Vault address can return any body it + likes. It must neither be buffered whole nor shown to the user.""" + marker = "SENSITIVE-INTERNAL-DETAIL" + stub_vault.sign_error_body = (marker + "x" * 64).encode() * 200_000 + + user, target = make_user_and_target(api, cert_ssh_port) + # With a PTY, or the central assertion is unfalsifiable: without a PTY + # channel `emit_pty_output` has nothing to write to, so *no* connection + # error reaches the client and "the marker is absent" holds however + # badly the body is handled. The sanitiser could be removed entirely and + # this would still pass. + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert stub_vault.signs, "the signing request was never made" + assert client.returncode != 0 + # Something was shown, and it was not the issuer's own words. + assert "Vault" in shown or "certificate" in shown.lower(), ( + f"no failure reached the client at all: {shown[:200]!r}" + ) + assert marker not in shown + + def test_an_oversized_signing_response_is_not_buffered_whole( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The failed-response path was bounded; the successful one is the same + body with a different status code on it, and it is parsed once per + session. Left unbounded it is memory the issuer gets to allocate inside + Warpgate, as often as sessions are started.""" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + gateway = psutil.Process(cert_wg.process.pid) + before = gateway.memory_info().rss + stub_vault.signed_key = "ssh-ed25519-cert-v01@openssh.com " + "A" * 100_000_000 + assert connect(processes, cert_wg, user, target, timeout * 3)[0] != 0 + growth = gateway.memory_info().rss - before + + assert growth < 50_000_000, f"the body was buffered whole ({growth / 1e6:.0f} MB)" + + # And the gateway is still there, with the memory it started with. + stub_vault.signed_key = None + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + def test_a_revoked_token_costs_one_login_no_matter_how_many_sessions_find_it( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A Vault restart is discovered by every session in flight at the same + moment. If each answered by logging in, ordinary traffic would meet the + restart with a login per session — and each of those with a credential + read off disk.""" + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + stub_vault.invalidate_token() + # Wide enough that the sessions are genuinely inside the same login, + # rather than politely arriving one after another. + stub_vault.login_delay = 1 + logins = len(stub_vault.logins) + + clients = [start(processes, cert_wg, user, target) for _ in range(6)] + for client in clients: + client.communicate(timeout=timeout * 3) + + assert [client.returncode for client in clients] == [0] * 6 + assert len(stub_vault.logins) - logins == 1 + + def test_the_client_is_never_shown_the_issuers_own_words( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """What the user sees comes from a fixed list; the issuer's body is + written for operators and names mounts, policies and hosts. This asks + for a PTY on purpose — without one the message has nowhere to go and the + check would pass without anything being shown at all.""" + stub_vault.sign_error_body = b"1 error occurred: permission denied by policy ssh-signer-7" + user, target = make_user_and_target(api, cert_ssh_port) + code, stdout = connect(processes, cert_wg, user, target, timeout, "-tt") + + assert code != 0 + shown = stdout.decode(errors="replace") + assert "Vault service error" in shown, "the failure never reached the user" + assert "ssh-signer-7" not in shown + assert "policy" not in shown + + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" + + def test_an_error_body_split_mid_character_does_not_kill_the_session( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Truncating at a fixed byte count lands inside a multi-byte character + for some bodies; slicing a Rust `String` there panics.""" + offset = Path(cert_wg.log_path).stat().st_size + stub_vault.sign_error_body = b"a" * 255 + "é".encode() + b"b" * 64 + + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" + + # Everything above is satisfied whether or not the truncation is safe: + # the stub's 500 already fails the session, and `tokio::sync::Mutex` does + # not poison, so the next login succeeds even if a task panicked. + # + # Measured with the guard removed: the panic takes the signing task down + # and the client hangs until its own timeout, so what discriminates in + # practice is the first `connect` never returning. This assertion is the + # faster and more legible signal for the case where a panic does not + # hang — it is not the one that fires today, and saying so is cheaper + # than someone later assuming it is. + assert "panicked at" not in log_since(cert_wg, offset), ( + "slicing the body at a byte boundary panicked the signing task" + ) + + # The gateway has to still be there afterwards. + stub_vault.sign_error_body = None + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + def test_issuer_unreachable_fails_promptly( + self, processes: ProcessManager, ctx, cert_ssh_port, timeout + ): + """An unreachable Vault must fail the session rather than stall it, so + an issuer outage looks like an auth failure and not a hang.""" + dead = StubVault(ctx.tmpdir / f"dead-vault-{uuid4()}") + dead.start() + address = dead.url + dead.stop() + + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": address, + "default_role": "warpgate", + "timeout": "5s", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + # The assertion is the absence of a timeout: communicate() raises if the + # session is still open when the test's own deadline passes. + code, _ = connect(processes, wg, user, target, timeout) + assert code != 0 + + +class TestAuthMethods: + def test_kubernetes_sends_the_service_account_token( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + # An earlier test may have left a usable token cached, so force a login. + stub_vault.invalidate_token() + connect(processes, cert_wg, user, target, timeout) + + login = stub_vault.logins[-1] + assert login["method"] == "kubernetes" + assert login["role"] == "warpgate" + assert login["jwt"] == SERVICE_ACCOUNT_JWT + + def test_approle_sends_the_secret_id_from_its_file( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """The secret ID is read from disk rather than the config so it can be + rotated underneath a running Warpgate.""" + secret_id_path = ctx.tmpdir / f"secret-id-{uuid4()}" + secret_id_path.write_text("stub-secret-id\n") + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "app_role", + "role_id": "stub-role-id", + "secret_id_path": str(secret_id_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + login = stub_vault.logins[-1] + assert login["method"] == "approle" + assert login["role_id"] == "stub-role-id" + assert login["secret_id"] == "stub-secret-id" + + +class TestCloudAuthMethods: + """The cloud methods take their credential from a metadata service, so + nothing durable is written to the host at all. Only the request Warpgate + builds is under test here; the metadata services themselves need a real VM.""" + + def _wg_with_auth(self, processes, stub_vault, auth, env=None): + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": auth, + } + }, + env=env, + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + return wg + + def test_azure_sends_imds_token_and_vm_coordinates( + self, processes: ProcessManager, stub_vault, cert_ssh_port, timeout + ): + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "azure", + "role": "warpgate", + "metadata_address": stub_vault.url, + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + login = stub_vault.logins[-1] + assert login["method"] == "azure" + assert jwt_claims(login["jwt"])["aud"] == "https://management.azure.com/" + assert login["subscription_id"] == "sub-1" + assert login["resource_group_name"] == "rg-1" + assert login["vm_name"] == "vm-1" + assert any("management.azure.com" in r for r in stub_vault.metadata_requests) + + def test_gcp_requests_a_token_bound_to_its_role( + self, processes: ProcessManager, stub_vault, cert_ssh_port, timeout + ): + """The audience ties the token to one Vault role, so a token minted for + another role cannot be presented here.""" + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "gcp", + "role": "warpgate", + "metadata_address": stub_vault.url, + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + login = stub_vault.logins[-1] + assert login["method"] == "gcp" + assert jwt_claims(login["jwt"])["aud"] == "vault/warpgate" + assert any( + "audience=vault%2Fwarpgate" in r for r in stub_vault.metadata_requests + ) + + def test_aws_signs_the_global_endpoint_by_default( + self, processes: ProcessManager, stub_vault, cert_ssh_port, timeout + ): + """Vault replays the request against the global STS endpoint, which only + accepts signatures scoped to us-east-1. Signing a regional endpoint by + default makes every login fail with SignatureDoesNotMatch.""" + from base64 import b64decode + + wg = self._wg_with_auth( + processes, + stub_vault, + {"kind": "aws", "role": "warpgate"}, + env={ + "AWS_ACCESS_KEY_ID": "ASIA0000000000000000", + "AWS_SECRET_ACCESS_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "AWS_SESSION_TOKEN": "AQoDYXdzEJr1KEXAMPLEtoken", + }, + + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + connect(processes, wg, user, target, timeout) + + login = stub_vault.logins[-1] + assert b64decode(login["iam_request_url"]) == b"https://sts.amazonaws.com/" + + headers = json.loads(b64decode(login["iam_request_headers"])) + authorization = next( + value for name, value in headers.items() if name.lower() == "authorization" + ) + assert "/us-east-1/sts/aws4_request" in authorization + + def test_aws_sends_a_signed_sts_request( + self, processes: ProcessManager, stub_vault, cert_ssh_port, timeout + ): + """Vault replays the signed request against STS, so the signature — not a + disclosed credential — is what proves identity.""" + from base64 import b64decode + + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "aws", + "role": "warpgate", + "region": "us-east-1", + "server_id": "vault.example.com", + }, + # Supplied here rather than inherited, so the test does not depend on + # whatever credentials the developer happens to have. The signature is + # verified for shape, never sent to AWS. + env={ + "AWS_ACCESS_KEY_ID": "ASIA0000000000000000", + "AWS_SECRET_ACCESS_KEY": "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY", + "AWS_SESSION_TOKEN": "AQoDYXdzEJr1KEXAMPLEtoken", + }, + + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + connect(processes, wg, user, target, timeout) + + login = stub_vault.logins[-1] + assert login["method"] == "aws" + assert login["iam_http_request_method"] == "POST" + assert b64decode(login["iam_request_url"]) == b"https://sts.us-east-1.amazonaws.com/" + assert b64decode(login["iam_request_body"]) == ( + b"Action=GetCallerIdentity&Version=2011-06-15" + ) + + headers = json.loads(b64decode(login["iam_request_headers"])) + headers = {name.lower(): value for name, value in headers.items()} + assert headers["x-vault-aws-iam-server-id"] == "vault.example.com" + assert "AWS4-HMAC-SHA256" in headers["authorization"] + assert "x-vault-aws-iam-server-id" in headers["authorization"], ( + "the server ID must be signed, not merely sent" + ) + + def test_approle_response_wrapping( + self, processes, cert_ssh_port, stub_vault, ctx, timeout + ): + secret_id_path = ctx.tmpdir / f"wrapping-token-{uuid4()}" + secret_id_path.write_text("unwrap:stub-wrapping-token") + + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "app_role", + "role_id": "role-1", + "secret_id_path": str(secret_id_path), + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + login = stub_vault.logins[-1] + assert login["method"] == "approle" + assert login["secret_id"] == "unwrapped-secret-id" + + def test_a_wrapping_token_is_redeemed_once_and_the_secret_id_reused( + self, processes, cert_ssh_port, stub_vault, ctx, timeout + ): + """A wrapping token can be redeemed exactly once, while the secret ID + inside it stays usable. Unwrapping per login would leave every session + after the first unable to authenticate to Vault at all.""" + secret_id_path = ctx.tmpdir / f"wrapping-token-{uuid4()}" + secret_id_path.write_text("unwrap:stub-wrapping-token") + + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "app_role", + "role_id": "role-1", + "secret_id_path": str(secret_id_path), + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + # Forces a second login rather than waiting out the lease. + stub_vault.invalidate_token() + assert connect(processes, wg, user, target, timeout)[0] == 0 + + assert len(stub_vault.logins) == 2, "the second session did not log in again" + assert stub_vault.logins[-1]["secret_id"] == "unwrapped-secret-id" + assert len(stub_vault.unwraps) == 1, "the wrapping token was redeemed twice" + + def test_a_freshly_provisioned_wrapping_token_is_picked_up( + self, processes, cert_ssh_port, stub_vault, ctx, timeout + ): + """Caching the unwrapped secret ID must not mean ignoring the file: an + operator rotating the credential writes a new wrapping token there.""" + secret_id_path = ctx.tmpdir / f"wrapping-token-{uuid4()}" + secret_id_path.write_text("unwrap:first-wrapping-token") + + wg = self._wg_with_auth( + processes, + stub_vault, + { + "kind": "app_role", + "role_id": "role-1", + "secret_id_path": str(secret_id_path), + }, + ) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + secret_id_path.write_text("unwrap:second-wrapping-token") + stub_vault.invalidate_token() + assert connect(processes, wg, user, target, timeout)[0] == 0 + + assert stub_vault.unwraps == ["first-wrapping-token", "second-wrapping-token"] + + +class TestConfigReload: + def test_a_new_vault_address_takes_effect_without_a_restart( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """Every other section of the config is watched and applied live. A + `vault:` that quietly needed a restart would be the one exception, and + an operator editing it would have no way of knowing.""" + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + second = StubVault(ctx.tmpdir / f"stub-vault-reload-{uuid4()}") + second.start() + try: + port = processes.start_ssh_server(trusted_ca=[second.ca_public_key]) + wait_port(port) + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, port) + + # The running config points at the first issuer, whose CA this + # target does not trust — so this must fail before the edit. + assert connect(processes, wg, user, target, timeout)[0] != 0 + + config = yaml.safe_load(wg.config_path.read_text()) + config["vault"]["address"] = second.url + wg.config_path.write_text(yaml.dump(config)) + + # The watcher debounces for 500ms before reloading once. + deadline = time.time() + 20 + while time.time() < deadline: + if second.signs: + break + connect(processes, wg, user, target, timeout) + time.sleep(1) + + assert second.signs, "the edited Vault address was never picked up" + assert connect(processes, wg, user, target, timeout)[0] == 0 + finally: + second.stop() + + +class TestTheHostKeyCheck: + """The admin host-key check reaches the same connection code the SSH path + does. It asks for one thing — the target's host key — and must not carry on + into authentication behind the operator's back.""" + + def test_checking_a_host_key_issues_no_certificate( + self, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Pressing the button on a certificate target would otherwise mint a + real certificate and open a real authenticated session that nothing is + attached to, holding until the inactivity timeout.""" + _, target = make_user_and_target(api, cert_ssh_port) + + first = api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + assert first.remote_key_base64, "the check did not reach the target at all" + + # The first press leaves the key trusted; it is every press after that + # one which used to run on into authentication. + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + + # The request returns as soon as the key arrives; a connection that kept + # going would reach the issuer a moment afterwards, so the assertion has + # to outlast the response rather than race it. + deadline = time.time() + 5 + while time.time() < deadline: + assert stub_vault.signs == [], "the host key check issued a certificate" + time.sleep(0.25) + + def test_the_check_leaves_no_session_behind( + self, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """A session opened by the check is invisible in the UI and outlives the + request, so counting what the gateway is still holding is the only way + to see it.""" + _, target = make_user_and_target(api, cert_ssh_port) + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + + # Measured as a delta: the gateway is shared by the whole module, so + # earlier tests have left sessions and sockets of their own behind. + gateway = psutil.Process(cert_wg.process.pid) + target_socket_count = lambda: len( + [c for c in gateway.net_connections(kind="tcp") if c.raddr and c.raddr.port == cert_ssh_port] + ) + before_sockets = target_socket_count() + before_sessions = api.get_sessions().total + + for _ in range(3): + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + + assert api.get_sessions().total == before_sessions, "the check registered a session" + # A connection still open to the target is the observable trace of a + # client task that never exited. One-sided on purpose: an earlier test's + # socket finishing its close during this window lowers the count, and + # that is not what is under test. + assert target_socket_count() <= before_sockets + + +class TestTheStubItself: + """The auth-method tests only mean something if the stub would have noticed a + wrong payload. These check the checker, without starting anything.""" + + def _aws_payload(self, **overrides): + from base64 import b64encode + + payload = { + "iam_http_request_method": "POST", + "iam_request_url": b64encode(b"https://sts.amazonaws.com/").decode(), + "iam_request_body": b64encode( + b"Action=GetCallerIdentity&Version=2011-06-15" + ).decode(), + "iam_request_headers": b64encode( + json.dumps({"authorization": "AWS4-HMAC-SHA256 Credential=..."}).encode() + ).decode(), + } + payload.update(overrides) + return payload + + def test_a_well_formed_aws_payload_is_accepted(self): + assert reject_login("aws", self._aws_payload()) is None + + @pytest.mark.parametrize( + "overrides", + [ + {"iam_http_request_method": "GET"}, + {"iam_request_url": "aHR0cHM6Ly9ldmlsLmV4YW1wbGUuY29tLw=="}, # not STS + {"iam_request_body": "QWN0aW9uPUFzc3VtZVJvbGU="}, # not GetCallerIdentity + {"iam_request_headers": "e30="}, # {} — unsigned + {"iam_request_url": "not base64 at all"}, + ], + ) + def test_a_mangled_aws_payload_is_rejected(self, overrides): + assert reject_login("aws", self._aws_payload(**overrides)) is not None + + def test_a_non_jwt_identity_token_is_rejected(self): + assert reject_login("kubernetes", {"role": "warpgate", "jwt": "a-token"}) + assert reject_login("kubernetes", {"role": "warpgate", "jwt": SERVICE_ACCOUNT_JWT}) is None + + def test_a_gcp_token_for_another_role_is_rejected(self): + from .stub_vault import jwt + + assert reject_login( + "gcp", {"role": "warpgate", "jwt": jwt({"aud": "vault/other-role"})} + ) + assert ( + reject_login("gcp", {"role": "warpgate", "jwt": jwt({"aud": "vault/warpgate"})}) + is None + ) + + def test_azure_coordinates_are_all_required(self): + from .stub_vault import jwt + + complete = { + "role": "warpgate", + "jwt": jwt({"aud": "https://management.azure.com/"}), + "subscription_id": "sub-1", + "resource_group_name": "rg-1", + "vm_name": "vm-1", + } + assert reject_login("azure", complete) is None + for field in ("subscription_id", "resource_group_name", "vm_name"): + assert reject_login("azure", {**complete, field: ""}), f"{field} not checked" + + +class TestControlsStillApply: + """A new authentication path is the classic place for an existing control to + go missing — the shape of GHSA-qmr2-wp96-h9ff.""" + + def test_an_unauthorized_user_never_reaches_the_issuer( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port, assign=False) + code, _ = connect(processes, cert_wg, user, target, timeout) + + assert code != 0 + assert stub_vault.signs == [], "a certificate was issued before authorization" + + def test_a_role_that_climbs_out_of_the_mount_never_leaves_the_process( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The role is put into the request path, and a URL is normalised before + it is sent: `../../auth/token/create` would arrive at a different Vault + API altogether, with the gateway's own token attached to it. + + Refused when the target is saved, which is the first moment anyone can + be told. It used to be refused only when a session tried to use it, so + the admin learned of it from a broken connection hours later. + + The signing path still refuses it — `validate_segment` in + `warpgate-vault`, held by `test_segment_validation` — and that layer is + now unreachable through the API, which is the point of refusing early. + Both layers read the same rule from `warpgate_common`, so they cannot + drift into disagreeing about what a role may be called. + """ + with pytest.raises(sdk.ApiException) as refused: + make_user_and_target(api, cert_ssh_port, role="../../auth/token/create") + assert refused.value.status == 400, refused.value.status + assert stub_vault.requests == [], "a request left the process" + + # Which only means something if a target that names a real role does + # reach Vault from this same gateway. + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + assert stub_vault.requests + + def test_no_certificate_is_issued_for_targets_using_other_auth( + self, processes, cert_wg, stub_vault, api, timeout, wg_c_ed25519_pubkey + ): + port = processes.start_ssh_server( + trusted_keys=[wg_c_ed25519_pubkey.read_text()] + ) + wait_port(port) + + wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user(sdk.CreateUserRequest(username=f"user-{uuid4()}")) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, wg_role.id) + target = api.create_target( + sdk.TargetDataRequest( + name=f"pubkey-{uuid4()}", + options=sdk.TargetOptions( + sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host=TARGET_HOST, + port=port, + username="root", + auth=sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetPublicKeyAuth(kind="PublicKey") + ), + ) + ), + ) + ) + api.add_target_role(target.id, wg_role.id) + + connect(processes, cert_wg, user, target, timeout) + assert stub_vault.signs == [] + + +class TestSecrets: + def test_the_vault_token_is_never_logged( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + user, target = make_user_and_target(api, cert_ssh_port) + connect(processes, cert_wg, user, target, timeout) + + assert stub_vault.valid_token + log = Path(cert_wg.log_path).read_text() + # Asserted so the check below cannot pass merely because nothing was logged. + assert "Issued an SSH certificate" in log + assert stub_vault.valid_token not in log + + def test_no_credential_reaches_the_log_even_at_trace_level( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """Trace is where an operator goes when sessions will not connect, so it + is the setting most likely to be on while something is going wrong — and + the output most likely to be pasted into an issue. Neither the + credential Warpgate authenticates with nor the token it gets back may be + in it, at any verbosity.""" + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + + log_path = ctx.tmpdir / f"trace-wg-{uuid4()}.log" + with log_path.open("w") as log: + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + }, + env={"RUST_LOG": "trace"}, + stdout=log, + stderr=log, + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, wg, user, target, timeout)[0] == 0 + + text = Path(log_path).read_text(errors="replace") + # Both halves of the exchange happened, so the log had the chance. + assert "Issued an SSH certificate" in text + assert stub_vault.logins and stub_vault.valid_token + + assert stub_vault.valid_token not in text, "the Vault token is in the log" + assert SERVICE_ACCOUNT_JWT not in text, "the service account token is in the log" + assert SERVICE_ACCOUNT_JWT.split(".")[1] not in text, "its claims are in the log" + + def test_no_ephemeral_key_is_stored( + self, processes, cert_wg, cert_ssh_port, api, timeout + ): + """The whole point of the feature: nothing the target would trust may + outlive the connection.""" + before = len(api.get_ssh_own_keys()) + + user, target = make_user_and_target(api, cert_ssh_port) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + assert len(api.get_ssh_own_keys()) == before + + +class TestAChainWithAJumpHost: + """Every other test here uses exactly one hop. + + Composition is where identity gets confused — whose key, whose certificate, + whose account — and the code has a chain resolver that reverses its list. + Not testing it is how the host-key check came to report the jump host's key + as the target's. + """ + + def _chain(self, api, jump_port, target_port, extensions=None): + """A target reached through a jump host, both on certificate auth.""" + wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user(sdk.CreateUserRequest(username=f"user-{uuid4()}")) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, wg_role.id) + + def make(name, port, jump=None, host=TARGET_HOST, extensions=None): + options = sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host=host, + port=port, + username="root", + auth=sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetCertificateAuth( + kind="Certificate", + role=None, + allowed_critical_options=[], + allowed_extensions=extensions or ["permit-pty"], + ) + ), + ) + if jump is not None: + options.jump_host = jump + target = api.create_target( + sdk.TargetDataRequest(name=name, options=sdk.TargetOptions(options)) + ) + api.add_target_role(target.id, wg_role.id) + return target + + # A target used as a jump host needs `permit-port-forwarding` on its own + # certificate: Warpgate reaches the next hop by opening a direct-tcpip + # channel through it, and OpenSSH judges that purely on what the + # certificate carries. Naming it here is the point of the allow-list — + # before it existed every certificate carried this silently. + jump = make( + f"jump-{uuid4()}", + jump_port, + extensions=["permit-pty", "permit-port-forwarding"], + ) + # Dialled from inside the jump host's container, where `localhost` is + # the container itself. + target = make( + f"behind-{uuid4()}", + target_port, + jump=jump.id, + host="host.docker.internal", + extensions=extensions, + ) + return user, jump, target + + def _two_fresh_hops(self, processes, api, stub_vault): + """A chain whose two hops have never been seen by anything. + + Freshly started, so neither key is trusted whatever ran before — the + jump host used to be the file's shared fixture server, which earlier + tests connect to, and a test whose subject is *whether a key is trusted* + cannot borrow a server whose key another test has already trusted. + + Each hop gets a key of its own too: two servers sharing one are + indistinguishable to exactly the thing under test. + """ + stub_vault.sign_options = ["permit-port-forwarding"] + jump_port = processes.start_ssh_server( + trusted_ca=[stub_vault.ca_public_key], distinct_host_key=True + ) + target_port = processes.start_ssh_server( + trusted_ca=[stub_vault.ca_public_key], distinct_host_key=True + ) + wait_port(jump_port) + wait_port(target_port) + assert processes.host_keys[jump_port] != processes.host_keys[target_port], ( + "the two hops were started with the same key, so nothing below " + "distinguishes them" + ) + return (jump_port, target_port, *self._chain(api, jump_port, target_port)) + + def test_an_untrusted_jump_host_is_refused_rather_than_traversed( + self, processes, stub_vault, api + ): + """Checking the target's host key goes through the jump host, whose own + key nothing has trusted yet. Accepting it there would trust a host on + the strength of a question asked about a different one.""" + jump_port, _, _, _, target = self._two_fresh_hops(processes, api, stub_vault) + before = len(stub_vault.signs) + + with pytest.raises(sdk.ApiException) as refused: + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + # The message, not just the failure. Without the refusal the hop's key + # is declined at the transport instead and the endpoint answers "SSH + # protocol error" — a failure either way, which is why asserting only + # that this raises would be evidence for nothing. + assert "untrusted host key" in str(refused.value.body), refused.value.body + + # Refused, and refused early: nothing was authenticated on the way, so + # no certificate was minted for anyone. + assert len(stub_vault.signs) == before, "a certificate was issued anyway" + # And the hop was not quietly trusted in passing, which is the failure + # this refusal exists to prevent. + assert not [ + host for host in api.get_ssh_known_hosts() if host.port == jump_port + ], "the jump host's key was trusted on the way through" + + def test_the_host_key_check_reports_the_target_and_not_the_jump_host( + self, processes, stub_vault, api + ): + """Both hops present a key, and the endpoint used to answer with the + first one it saw. An operator pinning what they are told is the target's + key was pinning the jump host's, and the target's own key was never + looked at.""" + jump_port, target_port, _, _, target = self._two_fresh_hops( + processes, api, stub_vault + ) + + # The jump host is trusted here the way the admin UI trusts one — by + # recording its key — rather than by opening a session to it. Trusting + # it by connecting would make this test depend on the host-key + # verification mode, a single global parameter that any test sharing + # this gateway can change, and the point of this one is to depend on + # nothing but its own two servers. + jump_key = processes.host_keys[jump_port] + api.add_ssh_known_host( + sdk.AddSshKnownHostRequest( + host=TARGET_HOST, + port=jump_port, + key_type=jump_key.key_type, + key_base64=jump_key.base64, + ) + ) + + reported = api.check_ssh_host_key( + sdk.CheckSshHostKeyRequest(target_id=target.id) + ) + + # By identity, not by elimination. "Not the jump host's key" and "the + # target's key" are the same claim for a chain of two and stop being the + # same for a chain of three, and the weaker one is what this test made + # until an outside verifier read it (W-119). + target_key = processes.host_keys[target_port] + assert reported.remote_key_base64 != jump_key.base64, ( + "checking the target answered with the jump host's key" + ) + assert (reported.remote_key_type, reported.remote_key_base64) == ( + target_key.key_type, + target_key.base64, + ), "the reported key is not the one the target was started with" + + def test_checking_a_chained_target_authenticates_only_to_the_jump_host( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Reaching the target's transport means authenticating to the jump + host first — there is no tunnel otherwise — so one certificate is + minted, for the hop that is traversed. The target itself is stopped at + its host key, before anything is offered to it.""" + stub_vault.sign_options = ["permit-port-forwarding"] + second = processes.start_ssh_server( + trusted_ca=[stub_vault.ca_public_key], distinct_host_key=True + ) + wait_port(second) + + user, jump, target = self._chain(api, cert_ssh_port, second) + assert connect(processes, cert_wg, user, jump, timeout)[0] == 0 + + before = len(stub_vault.signs) + api.check_ssh_host_key(sdk.CheckSshHostKeyRequest(target_id=target.id)) + minted = stub_vault.signs[before:] + + assert len(minted) == 1, f"expected one certificate for the jump host, got {len(minted)}" + + # And it has to name whoever asked. There is no session to look up here + # — a button press is not a login — so the key ID used to fall back to + # the random UUID that stood in for one, and both the jump host's sshd + # log and Vault's issuance log recorded a certificate resolving to + # nobody. + # + # `admin-token`, not `admin`: this suite authenticates with an API + # token, which carries no username, and the first version of the fix + # substituted the literal string "admin" for it — recording an API call + # as though a person by that name had opened the session. That is the + # same attribution failure one layer along, so the label says what it + # actually was. + key_id = minted[0]["key_id"] + assert key_id.startswith("warpgate:admin-token:"), ( + f"the certificate misnames who asked: {key_id}" + ) + + def test_a_session_through_a_jump_host_works( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The chain still has to work, and each hop gets its own certificate + naming its own account.""" + # The stub signs one set of options for every request, where real Vault + # would use a role per target. So the leaf has to permit what the jump + # host needs; the two host-key checks above, which traverse the jump + # host on the leaf's own allow-list, are where it does its work. + stub_vault.sign_options = ["permit-port-forwarding"] + second = processes.start_ssh_server(trusted_ca=[stub_vault.ca_public_key]) + wait_port(second) + + user, _, target = self._chain( + api, cert_ssh_port, second, extensions=["permit-pty", "permit-port-forwarding"] + ) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + assert len(stub_vault.signs) == 2, "each hop needs its own certificate" + assert all(sign["valid_principals"] == "root" for sign in stub_vault.signs) diff --git a/tests/test_vault_contract.py b/tests/test_vault_contract.py new file mode 100644 index 000000000..56d547b2d --- /dev/null +++ b/tests/test_vault_contract.py @@ -0,0 +1,299 @@ +"""Warpgate against a real Vault and a real OpenBao. + +`test_ssh_target_cert_auth.py` runs against a stub, which is fast and can be made +to misbehave on demand — but it only knows what we told it. Every behaviour it +models is a claim about the real server, and the claims have been wrong twice: +a wrapping token treated as reusable, and a `lease_duration` of zero read as +expiry. Both defects were invisible for exactly as long as the stub was the only +witness. + +So each such claim is pinned here, once, against the thing itself. The +assertions read the request out of the server's own audit device — the payload +as the server received it, rather than as our stub chose to remember it. +""" + +import shutil +import subprocess +import time +from pathlib import Path +from uuid import uuid4 + +import pytest + +from .api_client import admin_client, sdk +from .conftest import ProcessManager +from .util import wait_port +from .vault_server import RealVault, matrix + +pytestmark = pytest.mark.skipif( + shutil.which("docker") is None, reason="needs Docker" +) + + +@pytest.fixture(params=matrix(), ids=lambda image: image.replace("/", "-").replace(":", "-")) +def server(request, ctx): + """A real issuer. Parametrised because OpenBao is not a rename of Vault: + it already differs on how an audit device may be enabled, and its cloud auth + methods are separate plugins rather than builtins.""" + vault = RealVault(request.param, config_dir=ctx.tmpdir / f"bao-{uuid4()}") + vault.start() + yield vault + vault.stop() + + +def start_warpgate(processes: ProcessManager, ctx, server: RealVault, *, wrapped=False): + secret_id_path = server.write_secret_id( + ctx.tmpdir / f"secret-id-{uuid4()}", wrapped=wrapped + ) + wg = processes.start_wg( + config_patch={ + "vault": { + "address": server.url, + "default_role": "warpgate", + "auth": { + "kind": "app_role", + "role_id": server.role_id, + "secret_id_path": str(secret_id_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + return wg, secret_id_path + + +def make_user_and_target(api, ssh_port, username="root"): + from .test_ssh_target_cert_auth import make_user_and_target as make + + return make(api, ssh_port, username=username) + + +def connect(processes, wg, user, target, timeout): + from .test_ssh_target_cert_auth import connect as do_connect + + return do_connect(processes, wg, user, target, timeout) + + +class TestAgainstARealIssuer: + def test_a_session_authenticates_end_to_end( + self, processes: ProcessManager, ctx, server, timeout + ): + """The whole point, against the real thing: no stored key on the + gateway, no authorized_keys on the target, a certificate the target + trusts because it trusts the CA.""" + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, _ = start_warpgate(processes, ctx, server) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port) + + assert connect(processes, wg, user, target, timeout) == (0, b"/bin/sh\n") + + # Read back out of the server's audit device, so this asserts on the + # request as it arrived rather than on anything we recorded ourselves. + assert server.signs, "the issuer never saw a signing request" + signed = server.signs[-1] + assert signed["valid_principals"] == "root" + assert signed["cert_type"] == "user" + assert signed["key_id"].startswith("warpgate:") + + def test_a_wrapping_token_cannot_be_redeemed_twice( + self, processes: ProcessManager, ctx, server, timeout + ): + """The claim the stub used to get wrong. Warpgate unwraps once and + reuses the secret ID; unwrapping per login would fail every login after + the first, and the server is the only witness that can prove it.""" + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, _ = start_warpgate(processes, ctx, server, wrapped=True) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port) + + for _ in range(3): + assert connect(processes, wg, user, target, timeout)[0] == 0 + + assert len(server.unwraps) == 1, "the wrapping token was redeemed more than once" + + def test_the_role_refuses_a_principal_it_does_not_allow( + self, processes: ProcessManager, ctx, server, timeout + ): + """`allowed_users` is the coarse gate that stands even when Warpgate is + wrong about who may reach what. `nobody` is outside it. + + Read from what the server *returned*, not from what it was asked. A + failed session and a recorded request are equally true when the role + issues the certificate happily and the target rejects an account it does + not have — which is the same observable, and the reason the sibling test + below carries the same warning. + """ + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, _ = start_warpgate(processes, ctx, server) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port, username="nobody") + + signs_before = len(server.signs) + issued_before = len(server.issued) + + assert connect(processes, wg, user, target, timeout)[0] != 0 + assert len(server.signs) > signs_before, "the request never reached the issuer" + assert len(server.issued) == issued_before, ( + "the role issued a certificate for a principal its allowed_users " + "does not list — the refusal under test never happened" + ) + + def test_the_certificate_carries_the_principals_that_were_asked_for( + self, processes: ProcessManager, ctx, server, timeout + ): + """Warpgate refuses a certificate that names anything other than the + account being reached. That rule is only safe because the server returns + the requested set verbatim rather than widening it — which has to be read + out of what the server *returned*. + + The first version of this test read the request instead, so it re-checked + Warpgate's own message and would have passed whatever came back. + """ + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, _ = start_warpgate(processes, ctx, server) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port, username="deploy") + + # `deploy` is in the role's allowed_users but is not a user on the + # target image, so the session fails at the target — after a + # certificate has been issued, which is what this test reads. + connect(processes, wg, user, target, timeout) + + assert server.signs[-1]["valid_principals"] == "deploy" + + # And what came back, which is the half that matters. + assert server.issued, "the server recorded no certificate" + certificate = server.issued[-1]["signed_key"] + principals = subprocess.run( + ["ssh-keygen", "-L", "-f", "-"], + input=certificate.encode(), + capture_output=True, + check=True, + ).stdout.decode() + principals = principals.split("Principals:")[1].split("Critical")[0].split() + assert principals == ["deploy"], f"the server returned {principals}" + + def test_a_restart_does_not_strand_the_gateway( + self, processes: ProcessManager, ctx, server, timeout + ): + """A dev-mode server loses everything on restart, so the cached token + stops being accepted before its lease runs out. Warpgate has to notice + and log in again rather than fail until the lease expires — the + behaviour the 403 re-login exists for, against a server that really + does forget.""" + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + + wg, secret_id_path = start_warpgate(processes, ctx, server) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] == 0 + + ca_before = server.ca_public_key + server.stop() + server.start() + # Dev mode regenerates everything, so the target's trust and the + # gateway's credential both have to be re-pointed. What is under test is + # that Warpgate recovers within a session, not that it survives a CA + # change nobody told it about. + assert server.ca_public_key != ca_before + server.write_secret_id(secret_id_path) + + ssh_port = processes.start_ssh_server(trusted_ca=[server.ca_public_key]) + wait_port(ssh_port) + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, ssh_port) + + deadline = time.time() + 30 + while time.time() < deadline: + if connect(processes, wg, user, target, timeout)[0] == 0: + break + time.sleep(1) + else: + pytest.fail("the gateway never recovered from the issuer restarting") + + +class TestTheStubMatchesTheServer: + """The stub's job is to stand in for these servers. Where it disagrees with + them, the stub is wrong — and every disagreement so far has hidden a defect + in Warpgate rather than in the stub.""" + + def test_a_token_with_no_lease_reports_zero(self, server): + """`lease_duration: 0` is how a token without a lease is reported, which + Warpgate used to read as "expired 30 seconds ago" and answer with a + fresh login on every certificate request.""" + response = server._api( + "POST", + "auth/approle/login", + {"role_id": server.role_id, "secret_id": server.secret_id}, + token=None, + ) + assert "lease_duration" in response["auth"] + assert isinstance(response["auth"]["lease_duration"], int) + + def test_a_wrapping_token_is_single_use(self, server): + """Asserted directly against the server, so the stub's single-use + modelling is a fact rather than a decision we made.""" + wrapping_token = server.wrapped_secret_id() + first = server._api("POST", "sys/wrapping/unwrap", {}, token=wrapping_token) + assert first["data"]["secret_id"] + + with pytest.raises(Exception): + server._api("POST", "sys/wrapping/unwrap", {}, token=wrapping_token) + + def test_a_key_id_is_refused_rather_than_substituted(self, server): + """With `allow_user_key_ids=false` the server errors on a request that + carries one, instead of quietly replacing it with the token's display + name. That is what makes a misconfigured role fail closed, and why + Warpgate needs no client-side check.""" + server._api( + "POST", + f"ssh-client-signer/roles/no-key-ids", + { + "key_type": "ca", + "allow_user_certificates": True, + "allowed_users": "root", + "allow_user_key_ids": False, + "ttl": "2m", + }, + ) + # A real key, because a malformed one is rejected at parse time and the + # role's policy is never consulted — which is how the first version of + # this test passed without exercising the thing it is named for. + public_key = Path("ssh-keys/id_ed25519.pub").read_text().strip() + + # The same request against the permissive role must succeed, or a + # failure below says nothing about `allow_user_key_ids`. + server._api( + "POST", + "ssh-client-signer/sign/warpgate", + { + "public_key": public_key, + "valid_principals": "root", + "cert_type": "user", + "key_id": "warpgate:alice:1234", + }, + ) + + with pytest.raises(Exception): + server._api( + "POST", + "ssh-client-signer/sign/no-key-ids", + { + "public_key": public_key, + "valid_principals": "root", + "cert_type": "user", + "key_id": "warpgate:alice:1234", + }, + ) diff --git a/tests/test_vault_hostile_certs.py b/tests/test_vault_hostile_certs.py new file mode 100644 index 000000000..4ba4a0a92 --- /dev/null +++ b/tests/test_vault_hostile_certs.py @@ -0,0 +1,470 @@ +"""Certificates a well-behaved issuer would never produce. + +Warpgate now checks what comes back — type, key, principals, critical options — +and each of those checks was added because something got through. These are the +shapes nobody has pointed at yet: absurd sizes, absurd validity, more principals +than anyone would name, a key ID as long as the certificate itself. + +None of these should authenticate. What is under test is that each fails as an +authentication failure, in bounded time, with the gateway still standing. +""" + +from uuid import uuid4 + +import pytest + +from .api_client import admin_client, sdk +from .conftest import ProcessManager, WarpgateProcess +from .stub_vault import SERVICE_ACCOUNT_JWT, StubVault +from .util import wait_port + +USER_PUBLIC_KEY_PATH = "ssh-keys/id_ed25519.pub" + + +@pytest.fixture(scope="module") +def stub_vault(ctx): + stub = StubVault(ctx.tmpdir / f"hostile-vault-{uuid4()}") + stub.start() + yield stub + stub.stop() + + +@pytest.fixture(scope="module") +def cert_wg(processes: ProcessManager, ctx, stub_vault): + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + return wg + + +@pytest.fixture(scope="module") +def cert_ssh_port(processes: ProcessManager, stub_vault): + port = processes.start_ssh_server(trusted_ca=[stub_vault.ca_public_key]) + wait_port(port) + return port + + +@pytest.fixture(autouse=True) +def reset_stub(stub_vault): + stub_vault.reset() + yield + stub_vault.reset() + + +@pytest.fixture +def api(cert_wg: WarpgateProcess): + with admin_client(f"https://localhost:{cert_wg.http_port}") as client: + yield client + + +def attempt(processes, cert_wg, api, cert_ssh_port, timeout, **target_kwargs): + from .test_ssh_target_cert_auth import connect, make_user_and_target + + user, target = make_user_and_target(api, cert_ssh_port, **target_kwargs) + return connect(processes, cert_wg, user, target, timeout) + + +class TestCertificatesNobodyShouldAccept: + def test_a_key_id_far_larger_than_the_certificate( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The key ID is chosen by Warpgate, but the issuer decides what it puts + in the certificate it returns. A 64 KiB one must not become 64 KiB in + the target's log — or in ours.""" + stub_vault.sign_key_id = "A" * 65536 + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert code != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" + + def test_a_certificate_with_a_different_key_id_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The key ID is the attribution, so a substituted one has to be caught. + + A plausible substitution rather than an absurd one: `sshd` accepts any + key ID at all and writes it to its log, so a target refuses nothing here + and a test asserting only a non-zero exit code proves nothing about our + check. The assertion is on Warpgate's own words. + + This test is the reason the matrix now verifies that a named + discriminator exists. The entry for this guard named exactly this + function for a week; it had never been written, and the guard was + reported on regardless. + """ + from .test_ssh_target_cert_auth import make_user_and_target, start + + stub_vault.sign_key_id = "warpgate:someone-else:00000000" + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert stub_vault.signs, "no certificate was ever requested" + assert client.returncode != 0, "a certificate naming another user was offered" + assert "Warpgate refused the certificate" in shown + assert "key ID other than the one requested" in shown + + def test_a_certificate_naming_a_thousand_principals( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """`root` is in there, so the target would accept it. Warpgate should + not: nobody asked for the other 999, and the set is meant to be what was + requested.""" + stub_vault.principals = ",".join([f"user{n}" for n in range(999)] + ["root"]) + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert stub_vault.signs, "no certificate was issued" + assert code != 0, "a certificate naming a thousand accounts was offered" + + def test_a_certificate_valid_for_a_century( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The whole point of the feature is a short window. A certificate good for + ten years defeats it.""" + from .test_ssh_target_cert_auth import make_user_and_target, start + + # `+36500d` is refused by ssh-keygen itself, so the stub used to crash and + # the session failed on "Vault is currently unavailable" — passing this + # test without a long-lived certificate ever existing. + stub_vault.validity = "-1d:+3650d" + user, target = make_user_and_target(api, cert_ssh_port) + + # Asked with a PTY and checked by *who* refused. A century-long + # certificate is one sshd is perfectly happy with, so asserting only + # that the session failed would pass on a rejection from the target — + # which is what the mutation matrix caught this test doing. + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert stub_vault.signs, "no certificate was issued" + assert client.returncode != 0, "a certificate valid for a century was accepted" + assert "Warpgate refused the certificate" in shown + assert "far longer than a session credential" in shown + + def test_a_certificate_carrying_a_hundred_critical_options( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Refusal has to survive the loop over them. + + Asked with a PTY and checked by *who* refused. These names are ones + OpenSSH does not recognise, so a target refuses the certificate on its + own — and asserting only a non-zero exit code passes just as well with + our own check deleted, which is what the certificate never reaching the + target would look like either way. + """ + from .test_ssh_target_cert_auth import make_user_and_target, start + + stub_vault.sign_options = [f"critical:opt{n}=v" for n in range(100)] + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert stub_vault.signs, "no certificate was ever requested" + assert client.returncode != 0, "a certificate carrying 100 options was accepted" + assert "Warpgate refused the certificate" in shown + assert "does not allow" in shown + + def test_a_signed_key_that_is_not_a_certificate_at_all( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.signed_key = "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIHNvbWV0aGluZw== not-a-cert" + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert code != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" + + def test_a_signed_key_that_is_a_megabyte_of_base64( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + stub_vault.signed_key = "ssh-ed25519-cert-v01@openssh.com " + ("A" * 1024 * 1024) + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert code != 0 + # Without this the test passes just as well when the certificate path + # never ran at all. + assert stub_vault.signs, "no certificate was ever requested" + + def test_the_gateway_survives_all_of_it( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """Runs last: after everything above, an ordinary session must still + work. A panic in any of those paths would show up here.""" + code, stdout = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + assert (code, stdout) == (0, b"/bin/sh\n") + + +class TestWhatTheOperatorIsTold: + """The README warns that a target whose clock lags rejects a short-lived + certificate "with an error that does not say so". The container's clock + cannot be moved without `SYS_TIME`, but the condition sshd sees is the same + one a certificate outside its window produces — and what the person + connecting is told is worth asserting, since that is what they debug from.""" + + def test_a_certificate_that_is_not_yet_valid( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + from .test_ssh_target_cert_auth import make_user_and_target, start + + stub_vault.validity = "+1h:+2h" + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + stdout = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert client.returncode != 0 + assert stub_vault.signs, "no certificate was issued" + # The window and the hint are the whole point: "rejected by the target" + # on its own sends someone to check credentials that are perfectly fine. + assert "check the target's clock" in stdout + assert "valid from" in stdout + + +class TestPrincipalsThatCrossOtherPeoplesBugs: + """A principal is not only ours to interpret — the target's sshd parses it + too, and has been wrong about it. + + CVE-2026-35414: a comma inside a certificate principal breaks OpenSSH's + access control (8.5p1 through 9.7p1), because one validation function splits + on the comma and authenticates the first fragment while the next treats the + whole string as one name. A certificate naming `deploy,root` can therefore + land a session as root. + + Warpgate already refuses to *request* a principal with a comma in it. The + question here is what it does with one that comes *back*. + """ + + def test_a_certificate_naming_more_than_the_target_account_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """`root` is present, so a membership check passes and the session + would succeed. The second name is an account the target will also + accept this certificate for, chosen by whoever answered rather than by + the operator — and under `AuthorizedPrincipalsFile` it need not look + like a username at all.""" + stub_vault.principals = "root,deploy" + code, _ = attempt(processes, cert_wg, api, cert_ssh_port, timeout) + + assert stub_vault.signs, "no certificate was issued" + assert code != 0, "a certificate naming accounts nobody asked for was offered" + + def test_a_hostile_option_name_cannot_write_to_the_terminal( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The refusal message quotes the option name straight from the + certificate and is written to the user's terminal. Escape sequences in + it would be executed by the terminal, not displayed — the same class as + GHSA-3c3w, one layer down.""" + from .test_ssh_target_cert_auth import make_user_and_target, start + + stub_vault.sign_options = ["critical:\x1b[2J\x1b[1;31mHACKED=x"] + user, target = make_user_and_target(api, cert_ssh_port) + + client = start(processes, cert_wg, user, target, "-tt") + stdout = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert client.returncode != 0 + assert stub_vault.signs, "no certificate was issued" + # The negative assertion alone is satisfied by the message never + # arriving — a connection that dies before writing anything contains no + # escape sequences either. Every sibling in this file anchors on the + # refusal first; this one did not, and so proved only that *something* + # went wrong. The option name's printable tail pins the third case + # apart: the message reached the terminal, it named the option, and the + # escape was neutralised rather than the name being dropped. + assert "Warpgate refused the certificate" in stdout + # `HACKED`, not `HACKED=x`: `ssh-keygen` splits `critical:NAME=VALUE` on + # the first `=`, so the value is not part of the name the message + # quotes. Asserted wrongly once, and the run said so. + assert "HACKED" in stdout, "the refusal did not name the option at all" + # Neutralised rather than dropped. The escape has to be *present* in its + # inert form, or a fix that silently strips the option name would pass + # the two assertions above and the one below. + assert "\\u{1b}[2J" in stdout, "the escape was removed rather than escaped" + assert "\x1b[2J" not in stdout, "a certificate wrote escape sequences to the terminal" + + def test_a_pinned_option_that_is_absent_is_refused( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The allow-list was built against someone who *adds* an option nobody + asked for. Approached from the other side they remove an expected one, + and a target whose whole point is a pinned `force-command` would accept + a certificate carrying none — a full shell instead of the one command. + + Checking only what arrived can never see this, which is why every check + here needs its complement.""" + from .test_ssh_target_cert_auth import connect, make_user_and_target + + stub_vault.sign_options = [] + user, target = make_user_and_target( + api, + cert_ssh_port, + allowed_critical_options=[("force-command", "/usr/local/bin/backup")], + ) + code, stdout = connect(processes, cert_wg, user, target, timeout) + + assert stub_vault.signs, "no certificate was issued" + assert code != 0, "a certificate without the required option was accepted" + assert b"/bin/sh" not in stdout, "the session got a shell" + + def test_a_pinned_option_that_is_present_still_works( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The complement of the complement: requiring the option must not break + the case it was written for.""" + from .test_ssh_target_cert_auth import connect, make_user_and_target + + stub_vault.sign_options = ["force-command=echo expected-by-the-operator"] + user, target = make_user_and_target( + api, + cert_ssh_port, + allowed_critical_options=[ + ("force-command", "echo expected-by-the-operator") + ], + ) + + assert connect(processes, cert_wg, user, target, timeout) == ( + 0, + b"expected-by-the-operator\n", + ) + + +class TestTheCredentialFileItself: + """The file a credential is read from is as much an input as anything on the + wire — it is written by whatever provisions the host, which can be wrong.""" + + def test_a_credential_file_too_large_to_be_one_is_refused( + self, processes: ProcessManager, ctx, stub_vault, cert_ssh_port, timeout + ): + """Reading it would outgrow the buffer reserved for the login payload + and reintroduce the grow-and-copy leak that reservation exists to + prevent — silently, since nothing else would notice.""" + from .test_ssh_target_cert_auth import connect, make_user_and_target + + token_path = ctx.tmpdir / f"huge-token-{uuid4()}" + token_path.write_text("x" * (64 * 1024)) + + wg = processes.start_wg( + config_patch={ + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + + with admin_client(f"https://localhost:{wg.http_port}") as api: + user, target = make_user_and_target(api, cert_ssh_port) + + assert connect(processes, wg, user, target, timeout)[0] != 0 + + # `logins` records only payloads the stub accepted, so a request that + # was sent and then rejected leaves it empty too — which made the first + # version of this assertion true either way. `requests` records every + # path before any validation, so it can tell "never sent" from "sent + # and refused". + assert not any("/login" in path for path in stub_vault.requests), ( + "an oversized credential was sent to the issuer" + ) + + def test_a_very_long_username_cannot_flood_the_targets_log( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The key ID exists so the target's own log names a person. A username + long enough to bury that log defeats it just as surely as a wrong name, + and the check on the *returned* key ID cannot see it — that one compares + against what was asked for, so an oversized request matches itself.""" + from .test_ssh_target_cert_auth import connect, USER_PUBLIC_KEY_PATH + + wg_role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user( + sdk.CreateUserRequest(username="u" * 4000 + str(uuid4())) + ) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, wg_role.id) + target = api.create_target( + sdk.TargetDataRequest( + name=f"cert-{uuid4()}", + options=sdk.TargetOptions( + sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host="localhost", + port=cert_ssh_port, + username="root", + auth=sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetCertificateAuth( + kind="Certificate", role=None, allowed_critical_options=[] + ) + ), + ) + ), + ) + ) + api.add_target_role(target.id, wg_role.id) + + assert connect(processes, cert_wg, user, target, timeout)[0] != 0 + # Refused before it is sent, so no oversized key ID is signed either. + assert stub_vault.signs == [], "an oversized key ID was sent to the issuer" + + def test_a_certificate_that_never_expires( + self, processes, cert_wg, cert_ssh_port, stub_vault, api, timeout + ): + """The lifetime bound reads `valid_before_time()`, which `ssh-key` + documents as returning `None` when the value overflows `i64` — i.e. for + a certificate marked never-expiring. So the check that exists to refuse a + credential which outlives its session is skipped by the one input an + adversary would reach for first. + + `-V always:forever` is what produces it, and it is also what a role with + no TTL at all yields.""" + from .test_ssh_target_cert_auth import make_user_and_target, start + + stub_vault.validity = "always:forever" + user, target = make_user_and_target(api, cert_ssh_port) + + # `-tt`, like every sibling in this file. This was skipped on the + # reasoning that the target holds an interactive session open past the + # timeout — but the target never sees this certificate at all: Warpgate + # refuses it before offering it, and without a PTY there is no channel + # for `emit_pty_output` to write the refusal to, so the client waits on + # a session that will never open. The forty-five seconds were the + # client's own timeout, not the target's. + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=timeout)[0].decode(errors="replace") + + assert stub_vault.signs, "no certificate was issued" + assert client.returncode != 0, "a never-expiring certificate was accepted" + assert "Warpgate refused the certificate" in shown + # The specific refusal, not just any. The exit code and the absence of a + # shell are satisfied by every other way this connection can fail, so on + # their own they would have passed with the guard deleted. + assert "never expires" in shown + assert "/bin/sh" not in shown diff --git a/tests/test_vault_hostile_target.py b/tests/test_vault_hostile_target.py new file mode 100644 index 000000000..3f6be385f --- /dev/null +++ b/tests/test_vault_hostile_target.py @@ -0,0 +1,400 @@ +"""Warpgate against a target that is not a real SSH server. + +The rest of the suite treats the target as honest. This does not. A target runs +on a machine Warpgate does not own, and russh — the library Warpgate is the +*client* half of here — has published pre-authentication panics reachable from +the peer. A compromised host that can hang or crash the gateway takes down more +than its own session. + +Two things are under test throughout: the session fails in bounded time, and the +gateway is still able to serve the next one afterwards. +""" + +import shutil +import time +from uuid import uuid4 + +import psutil +import pytest + +from .api_client import admin_client, sdk +from .conftest import TARGET_HOST, ProcessManager +from .hostile_ssh_server import MODES, HostileSSHServer +from .stub_vault import SERVICE_ACCOUNT_JWT, StubVault +from .util import wait_port + +pytestmark = pytest.mark.skipif(shutil.which("docker") is None, reason="needs Docker") + +USER_PUBLIC_KEY_PATH = "ssh-keys/id_ed25519.pub" + + +@pytest.fixture(scope="module") +def stub_vault(ctx): + stub = StubVault(ctx.tmpdir / f"hostile-target-vault-{uuid4()}") + stub.start() + yield stub + stub.stop() + + +@pytest.fixture(scope="module") +def cert_wg(processes: ProcessManager, ctx, stub_vault): + token_path = ctx.tmpdir / f"sa-token-{uuid4()}" + token_path.write_text(SERVICE_ACCOUNT_JWT) + wg = processes.start_wg( + config_patch={ + # Short, so a handshake that never finishes is measurable inside a + # test rather than only in production. + + "vault": { + "address": stub_vault.url, + "default_role": "warpgate", + "auth": { + "kind": "kubernetes", + "role": "warpgate", + "token_path": str(token_path), + }, + } + } + ) + wait_port(wg.http_port, for_process=wg.process, recv=False) + wait_port(wg.ssh_port, for_process=wg.process) + return wg + + +@pytest.fixture(scope="module") +def honest_target(processes: ProcessManager, stub_vault): + """Kept alongside the hostile ones so each test can prove the gateway still + works afterwards. A crash would otherwise look like a passing rejection.""" + port = processes.start_ssh_server(trusted_ca=[stub_vault.ca_public_key]) + wait_port(port) + return port + + +@pytest.fixture +def api(cert_wg): + with admin_client(f"https://localhost:{cert_wg.http_port}") as client: + yield client + + +def target_on(api, port): + from .test_ssh_target_cert_auth import make_user_and_target + + return make_user_and_target(api, port) + + +# `silent_after_banner` is left out here and tested on its own: it is bounded by +# the 30s handshake deadline rather than failing immediately, which needs a +# longer client timeout than the rest of these want. +@pytest.mark.parametrize("mode", sorted(set(MODES) - {"silent_after_banner"})) +def test_a_hostile_target_cannot_hang_or_crash_the_gateway( + mode, processes, cert_wg, honest_target, stub_vault, api, timeout +): + from .test_ssh_target_cert_auth import connect + + server = HostileSSHServer(mode) + server.start() + try: + user, target = target_on(api, server.port) + + gateway = psutil.Process(cert_wg.process.pid) + rss_before = gateway.memory_info().rss + + started = time.time() + code, _ = connect(processes, cert_wg, user, target, timeout) + elapsed = time.time() - started + + assert server.connections > 0, f"the {mode} server was never reached" + assert code != 0, f"a session completed against a {mode} server" + assert elapsed < 60, f"{mode} held the session for {elapsed:.0f}s" + + # An unbounded read shows up here rather than in the exit code. + growth = gateway.memory_info().rss - rss_before + assert growth < 256 * 1024 * 1024, ( + f"{mode} grew the gateway by {growth // (1024 * 1024)} MiB" + ) + finally: + server.stop() + + # The gateway has to still work: a panic in the client task would show up + # here and nowhere else. + user, target = target_on(api, honest_target) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + +def test_a_target_that_stalls_the_handshake_is_given_up_on( + processes, cert_wg, honest_target, stub_vault, api, timeout +): + """A peer that completes TCP, sends a valid identification string and then + says nothing was previously held forever: russh bounds the length of that + string but not the time to send what follows, and the inactivity timeout + only starts once the session loop is running.""" + from .test_ssh_target_cert_auth import start + + server = HostileSSHServer("silent_after_banner") + server.start() + try: + user, target = target_on(api, server.port) + started = time.time() + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=90)[0].decode(errors="replace") + code = client.returncode + elapsed = time.time() - started + + assert server.connections > 0, "the stalling server was never reached" + assert code != 0 + # The configured inactivity timeout is 8s; the assertion is that the + # handshake is bounded by something at all. + # The handshake has its own 30s bound. Without it this is held for the + # inactivity timeout instead — five minutes by default, and however long + # an operator has raised it to. + assert elapsed < 60, f"the gateway waited {elapsed:.0f}s on a silent handshake" + finally: + server.stop() + + +def test_a_hostile_target_never_gets_a_certificate_it_can_keep( + processes, cert_wg, honest_target, stub_vault, api, timeout +): + """A certificate is minted before the target proves anything, so a hostile + host does receive one — bounded by its two-minute validity and by naming + only the account on that host. What it must not get is one that is useful + anywhere else.""" + from .test_ssh_target_cert_auth import connect + + signs_before = len(stub_vault.signs) + server = HostileSSHServer("garbage_after_banner") + server.start() + try: + user, target = target_on(api, server.port) + connect(processes, cert_wg, user, target, timeout) + finally: + server.stop() + + # Nothing is offered until the transport is up, so a server that never gets + # that far never sees the certificate at all. Measured as a delta: earlier + # tests in this module sign against the honest target. + assert len(stub_vault.signs) == signs_before, ( + "a certificate was issued to a target that never completed a handshake" + ) + + +def test_a_jump_host_that_never_opens_the_tunnel_is_given_up_on( + processes, cert_wg, honest_target, stub_vault, api, timeout +): + """The step that had no bound at all. + + A jump host completes its own handshake and authenticates, so every deadline + that exists is satisfied — and then the request to open a tunnel to the next + hop goes unanswered. The next hop's deadline is armed inside + `wait_for_connection`, which does not run until that tunnel exists, so + nothing was watching this at all: the hold lasted until the previous hop's + inactivity timeout, five minutes by default. + """ + from .stalling_jump_host import PASSWORD, StallingJumpHost + from .test_ssh_target_cert_auth import USER_PUBLIC_KEY_PATH, connect, start + + jump = StallingJumpHost().start() + try: + role = api.create_role(sdk.RoleDataRequest(name=f"role-{uuid4()}")) + user = api.create_user(sdk.CreateUserRequest(username=f"user-{uuid4()}")) + api.create_public_key_credential( + user.id, + sdk.NewPublicKeyCredential( + label="Public Key", + openssh_public_key=USER_PUBLIC_KEY_PATH.read_text().strip(), + ), + ) + api.add_user_role(user.id, role.id) + + def make(name, port, auth, jump_host=None): + options = sdk.TargetOptionsTargetSSHOptions( + kind="Ssh", + host=TARGET_HOST, + port=port, + username="root", + auth=auth, + ) + if jump_host is not None: + options.jump_host = jump_host + target = api.create_target( + sdk.TargetDataRequest(name=name, options=sdk.TargetOptions(options)) + ) + api.add_target_role(target.id, role.id) + return target + + # Password auth for the jump host: the step under test is the tunnel, and + # negotiating a certificate here would put paramiko's algorithm support + # in the middle of it. + stalling = make( + f"stalling-{uuid4()}", + jump.port, + sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetPasswordAuth(kind="Password", password=PASSWORD) + ), + ) + behind = make( + f"behind-{uuid4()}", + honest_target, + sdk.SSHTargetAuth( + sdk.SSHTargetAuthSshTargetCertificateAuth( + kind="Certificate", role=None, allowed_critical_options=[] + ) + ), + jump_host=stalling.id, + ) + + # The bound under test is 30s, longer than the timeout the rest of this + # module wants, so the client is given its own. + started = time.time() + client = start(processes, cert_wg, user, behind, "-tt") + client.communicate(timeout=120) + code = client.returncode + elapsed = time.time() - started + assert jump.tunnel_requested.wait(1), "the tunnel was never requested" + assert code != 0, "a session completed through a jump host that never answered" + # The deadline is 30s. Anything near the inactivity timeout means it was + # not the tunnel step that gave up. + assert elapsed < 60, f"the jump host held the session for {elapsed:.0f}s" + finally: + jump.stop() + + user, target = target_on(api, honest_target) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + +def test_a_target_that_answers_with_a_host_key_and_then_stalls_is_given_up_on( + processes, cert_wg, honest_target, stub_vault, api, timeout +): + """The handshake deadline pauses for a host key decision. It must resume. + + Warpgate stops its own 30s handshake bound while an unknown host key is + outstanding, because under `Prompt` that wait belongs to a person reading a + fingerprint rather than to the target. The first version of that pause never + ended: the deadline was pushed a year out and nothing brought it back. The + verification mode is not even known at that point in the code — it is read + later — so `AutoAccept`, which answers in microseconds with nobody waiting, + cancelled the bound just as thoroughly. + + This server sends the key exchange reply carrying its host key and then + nothing at all, so the transport never completes. The key is unknown — it is + generated per instance — and the mode is set to `AutoAccept` below so the + answer comes straight back: that is what lifts the pause, and the remaining + handshake is the target's again from there, which is the moment the bound + has to come back. + + Measured, not assumed. With the resume disabled and the mode left at its + default the test still passed, because under `Prompt` the answer never + arrives, the pause is never lifted, and the line under test never executes. + The verifier reported the guard as undiscriminated for exactly that reason. + """ + from .stalling_host_key_server import StallingHostKeyServer + from .test_ssh_target_cert_auth import connect, start + + # Set explicitly, and this is the whole reason the test was worthless before. + # The default is `Prompt`, which waits for a human on a stdin the harness has + # closed — so the answer never came back, the pause was never lifted, and the + # line under test never ran at all. The mutation was inert and the test + # passed either way. The docstring said "the fixture accepts it + # automatically"; nothing in the fixture did. + api.update_parameters( + sdk.ParameterUpdate( + ssh_host_key_verification=sdk.SshHostKeyVerificationMode.AUTOACCEPT + ) + ) + + server = StallingHostKeyServer() + server.start() + try: + user, target = target_on(api, server.port) + started = time.time() + client = start(processes, cert_wg, user, target, "-tt") + shown = client.communicate(timeout=180)[0].decode(errors="replace") + elapsed = time.time() - started + + assert server.connections > 0, "the stalling server was never reached" + assert server.key_delivered.wait(1), ( + "the server never got as far as sending its host key, so the pause " + "under test was never entered" + ) + assert client.returncode != 0 + + # Warpgate's own message, not merely a dropped connection: the target is + # unreachable for several reasons here and only one of them is the one + # being measured. + assert "never completed the handshake" in shown, shown[-400:] + + # The bound is 30s, and the inactivity timeout it must not fall through + # to is 300s. Anything past 90s means the pause never ended. + assert elapsed < 90, ( + f"the gateway waited {elapsed:.0f}s after the host key was accepted" + ) + finally: + server.stop() + + user, target = target_on(api, honest_target) + assert connect(processes, cert_wg, user, target, timeout)[0] == 0 + + +def test_break_glass_user_creation_does_not_depend_on_vault( + processes: ProcessManager, ctx, stub_vault +): + """A Vault outage must not lock the door it is needed to open. + + `create-user` and `recover-access` build their services through + `Services::new_without_vault`, so a `vault:` section that cannot even be + constructed does not stop an operator making an account. Without that, a + misconfigured or unreachable Vault takes down every target *and* the command + for getting back in — including the one target you would fix Vault from. + + The mount here contains a slash, which `validate_segment` refuses, so + `Services::new` fails outright before any network call. That is deliberate: + it fails the same way with Vault up or down, so the test needs no outage and + no timeout. + + `recover-access` takes the same path but asserts an interactive terminal + first, so it cannot be driven from a harness; `create-user` is the same + branch and is the one pinned here. + """ + import subprocess + + import yaml + + from .conftest import binary_path, cargo_root + + wg = processes.start_wg() + wait_port(wg.http_port, for_process=wg.process, recv=False) + config = yaml.safe_load(wg.config_path.open()) + wg.process.kill() + wg.process.wait() + + config["vault"] = { + "address": stub_vault.url, + "default_role": "warpgate", + # Rejected by `validate_segment`: a mount is one path segment. + "mount": "ssh/../../sys", + "auth": {"kind": "kubernetes", "role": "warpgate", "token_path": "/dev/null"}, + } + broken = wg.config_path.parent / f"break-glass-{uuid4()}.yaml" + with broken.open("w") as f: + yaml.safe_dump(config, f) + + username = f"locked-out-{uuid4()}" + result = subprocess.run( + [ + str(cargo_root / binary_path), + "--config", + str(broken), + "create-user", + username, + "--password", + "not-a-real-password", + ], + capture_output=True, + timeout=120, + ) + output = (result.stdout + result.stderr).decode(errors="replace") + + # Naming the failure, because "non-zero exit" would also pass if the binary + # were missing or the config unparseable — neither of which is this guard. + assert "invalid Vault role or mount name" not in output, output[-600:] + assert result.returncode == 0, output[-600:] diff --git a/tests/vault_server.py b/tests/vault_server.py new file mode 100644 index 000000000..e8572afb2 --- /dev/null +++ b/tests/vault_server.py @@ -0,0 +1,343 @@ +"""A real Vault (or OpenBao) server, for the tests the stub cannot honestly make. + +`stub_vault.py` is fast and can be made to misbehave on demand, but it only knows +what we told it. Every behaviour it models is a claim about the real server, and +two of those claims have already been wrong: that a wrapping token can be +redeemed twice, and that a `lease_duration` of zero means expiry. This module +exists so each such claim is pinned by one test against the thing itself. + +The read surface deliberately matches the stub's — `url`, `ca_public_key`, +`signs`, `logins` — so an assertion can be written once and pointed at either. +`signs` and `logins` come from Vault's own audit device, which records the +request as the *server* received it rather than as our stub chose to remember it. +""" + +import json +import subprocess +import time +import urllib.error +import urllib.request +import uuid +from pathlib import Path + +from .util import alloc_port + +VAULT_IMAGE = "hashicorp/vault:1.20" +# Pinned, not `latest`. This harness is the gate that says the stub tells the +# truth about a real server; a floating tag means the thing it was checked +# against is not the thing it checks against tomorrow. +OPENBAO_IMAGE = "openbao/openbao:2.5.0" + +# The versions the contract suite runs against by default: one current release +# of each server. `WARPGATE_VAULT_MATRIX=full` widens it to the older releases +# an operator may still be running, which is where a behaviour change upstream +# would show up — pinning to one version means never noticing one. +MATRIX = { + "quick": [VAULT_IMAGE, OPENBAO_IMAGE], + "full": [ + "hashicorp/vault:1.15", + "hashicorp/vault:1.18", + VAULT_IMAGE, + "openbao/openbao:2.4", + OPENBAO_IMAGE, + ], +} + + +def matrix() -> list[str]: + import os + + return MATRIX.get(os.environ.get("WARPGATE_VAULT_MATRIX", "quick"), MATRIX["quick"]) + + +MOUNT = "ssh-client-signer" +ROLE = "warpgate" +ROOT_TOKEN = "test-root-token" +FIXED_ROLE_ID = "warpgate-test-role-id" +AUDIT_PATH = "/tmp/audit.log" + + +class RealVault: + """A dev-mode server with the SSH secrets engine and AppRole ready to use. + + Dev mode keeps everything in memory and unseals itself, which is what makes + it usable per-test; it is also why nothing here is a model for production. + """ + + def __init__(self, image: str = VAULT_IMAGE, config_dir: Path | None = None): + self.image = image + self.is_openbao = "openbao" in image + self.port = alloc_port() + self.container = f"warpgate-e2e-vault-{uuid.uuid4().hex[:8]}" + self.config_dir = config_dir + self.role_id: str | None = None + self.secret_id: str | None = None + + @property + def url(self) -> str: + return f"http://127.0.0.1:{self.port}" + + def ensure_image(self): + """Fails, rather than skips, when the image cannot be had. + + Skipping was the wrong instinct: this suite is the gate that says the + stub tells the truth about a real server, so a run where neither real + server was reached must not be able to report success. A registry + outage should stop the build and say so. + """ + present = subprocess.run( + ["docker", "image", "inspect", self.image], capture_output=True, check=False + ) + if present.returncode == 0: + return + pull = subprocess.run( + ["docker", "pull", self.image], capture_output=True, check=False + ) + if pull.returncode != 0: + raise Exception( + f"cannot obtain {self.image}, so the contract suite would prove " + f"nothing: {pull.stderr.decode()[-300:]}" + ) + + def start(self): + self.ensure_image() + token_env = "BAO_DEV_ROOT_TOKEN_ID" if self.is_openbao else "VAULT_DEV_ROOT_TOKEN_ID" + listen_env = ( + "BAO_DEV_LISTEN_ADDRESS" if self.is_openbao else "VAULT_DEV_LISTEN_ADDRESS" + ) + command = [ + "docker", "run", "--rm", "-d", + "--name", self.container, + "--cap-add", "IPC_LOCK", + "-p", f"{self.port}:8200", + "-e", f"{token_env}={ROOT_TOKEN}", + "-e", f"{listen_env}=0.0.0.0:8200", + ] + + # OpenBao refuses `sys/audit/*` over the API — "use declarative, + # config-based audit device management instead" — so the device has to + # be declared in a config file the server is started with. Vault takes + # it either way, and the API call keeps that path exercised too. + if self.is_openbao: + if self.config_dir is None: + raise Exception("OpenBao needs a config_dir for its audit device") + self.config_dir.mkdir(parents=True, exist_ok=True) + # `type` and `path` are both required, and the device's own settings + # go in `options` — a bare `file_path` is silently ignored with only + # a warning in the log, which looks exactly like a working audit + # device that never writes anything. + (self.config_dir / "audit.hcl").write_text( + 'audit "file" {\n' + ' type = "file"\n' + ' path = "file/"\n' + " options = {\n" + f' file_path = "{AUDIT_PATH}"\n' + ' log_raw = "true"\n' + " }\n" + "}\n" + ) + self.config_dir.chmod(0o755) + command += ["-v", f"{self.config_dir}:/openbao/testconfig"] + command += [self.image, "server", "-dev", "-config=/openbao/testconfig"] + else: + command += [self.image] + + subprocess.run(command, check=True, capture_output=True) + self._wait_until_up() + self._configure() + + def stop(self): + subprocess.run( + ["docker", "rm", "-f", self.container], capture_output=True, check=False + ) + + def _wait_until_up(self, timeout=60): + deadline = time.time() + timeout + while time.time() < deadline: + try: + self._api("GET", "sys/health", token=None) + return + except Exception: + time.sleep(0.5) + raise Exception(f"{self.image} did not come up on {self.url}") + + def _api(self, method: str, path: str, payload=None, token=ROOT_TOKEN): + request = urllib.request.Request( + f"{self.url}/v1/{path}", + method=method, + data=json.dumps(payload).encode() if payload is not None else None, + ) + if token: + request.add_header("X-Vault-Token", token) + if payload is not None: + request.add_header("Content-Type", "application/json") + with urllib.request.urlopen(request, timeout=10) as response: + body = response.read() + return json.loads(body) if body else {} + + def _configure(self): + # Raw, so the audit log shows the payload Warpgate actually sent rather + # than an HMAC of it. Acceptable only because this server lives for the + # duration of one test and holds nothing real. OpenBao has already taken + # its device from the config file it was started with. + if not self.is_openbao: + self._api( + "PUT", + "sys/audit/file", + { + "type": "file", + "options": {"file_path": AUDIT_PATH, "log_raw": "true"}, + }, + ) + + self._api("POST", "sys/mounts/" + MOUNT, {"type": "ssh"}) + self._api("POST", f"{MOUNT}/config/ca", {"generate_signing_key": True}) + self._api( + "POST", + f"{MOUNT}/roles/{ROLE}", + { + "key_type": "ca", + "algorithm_signer": "default", + "allow_user_certificates": True, + # Never `*`: the role is the coarse gate that stands even if + # Warpgate is wrong about who may reach what. + "allowed_users": "root,deploy", + "allow_user_key_ids": True, + "default_extensions": {"permit-pty": ""}, + "ttl": "2m", + "max_ttl": "5m", + }, + ) + + # The policy is the point of the exercise, not scaffolding: Warpgate is + # meant to hold an identity that can ask for certificates and nothing + # else. An auth method cannot issue a root token anyway — the server + # refuses with "auth methods cannot create root tokens" — so a test that + # reached for one would not be testing the deployment anybody runs. + self._api( + "PUT", + "sys/policies/acl/warpgate", + { + "policy": ( + f'path "{MOUNT}/sign/*" {{ capabilities = ["create", "update"] }}\n' + 'path "sys/wrapping/unwrap" { capabilities = ["update"] }\n' + ) + }, + ) + + self._api("POST", "sys/auth/approle", {"type": "approle"}) + self._api( + "POST", + "auth/approle/role/warpgate", + {"token_policies": "warpgate", "secret_id_num_uses": 0, "token_ttl": "10m"}, + ) + # Pinned rather than read back, so that a server which has been + # restarted comes up answering to the same `role_id` the running + # Warpgate was configured with. Otherwise a restart test only proves + # that a stale configuration fails, which nobody doubted. + self._api( + "POST", "auth/approle/role/warpgate/role-id", {"role_id": FIXED_ROLE_ID} + ) + self.role_id = FIXED_ROLE_ID + self.secret_id = self._api( + "POST", "auth/approle/role/warpgate/secret-id", {} + )["data"]["secret_id"] + + def wrapped_secret_id(self, ttl="5m") -> str: + """A response-wrapped secret ID, redeemable exactly once.""" + request = urllib.request.Request( + f"{self.url}/v1/auth/approle/role/warpgate/secret-id", + method="POST", + data=b"{}", + ) + request.add_header("X-Vault-Token", ROOT_TOKEN) + request.add_header("X-Vault-Wrap-TTL", ttl) + request.add_header("Content-Type", "application/json") + with urllib.request.urlopen(request, timeout=10) as response: + return json.load(response)["wrap_info"]["token"] + + @property + def ca_public_key(self) -> str: + request = urllib.request.Request(f"{self.url}/v1/{MOUNT}/public_key") + with urllib.request.urlopen(request, timeout=10) as response: + return response.read().decode().strip() + + def write_secret_id(self, path: Path, wrapped=False) -> Path: + path.write_text( + f"unwrap:{self.wrapped_secret_id()}" if wrapped else str(self.secret_id) + ) + return path + + # --- the same read surface the stub offers ----------------------------- + + def _audit(self) -> list[dict]: + result = subprocess.run( + ["docker", "exec", self.container, "cat", AUDIT_PATH], + capture_output=True, + check=False, + ) + entries = [] + for line in result.stdout.decode(errors="replace").splitlines(): + try: + entries.append(json.loads(line)) + except ValueError: + continue + return entries + + def _requests_to(self, suffix: str) -> list[dict]: + """What Warpgate *sent*. Useful for asserting the payload it builds — + and useless for asserting what the server decided, which is a distinct + question that `_responses_from` answers.""" + seen = [] + for entry in self._audit(): + if entry.get("type") != "request": + continue + path = entry.get("request", {}).get("path", "") + if path.endswith(suffix) or suffix in path: + seen.append(entry["request"].get("data") or {}) + return seen + + def _responses_from(self, suffix: str) -> list[dict]: + """What the server *returned*. + + The distinction is not academic: a test asserting that the certificate + names the requested principals was reading the request, so it re-checked + Warpgate's own message and would have passed no matter what came back. + """ + seen = [] + for entry in self._audit(): + if entry.get("type") != "response": + continue + path = entry.get("request", {}).get("path", "") + if path.endswith(suffix) or suffix in path: + seen.append((entry.get("response") or {}).get("data") or {}) + return seen + + @property + def signs(self) -> list[dict]: + return self._requests_to(f"{MOUNT}/sign/") + + @property + def issued(self) -> list[dict]: + """The certificates the server actually handed back. + + Only the ones carrying a `signed_key`. The audit device records a + response for a refusal too, so counting every response answered "did the + server reply", not "did it issue" — and a test asserting that a role + refuses a principal outside its `allowed_users` passed on the refusal + being recorded, which is the same shape as the request-versus-response + confusion this property was added to fix. + """ + return [ + data + for data in self._responses_from(f"{MOUNT}/sign/") + if data.get("signed_key") + ] + + @property + def logins(self) -> list[dict]: + return self._requests_to("auth/approle/login") + + @property + def unwraps(self) -> list[dict]: + return self._requests_to("sys/wrapping/unwrap") diff --git a/warpgate-admin/src/api/ssh_connection_test.rs b/warpgate-admin/src/api/ssh_connection_test.rs index 78905051a..cf374efd3 100644 --- a/warpgate-admin/src/api/ssh_connection_test.rs +++ b/warpgate-admin/src/api/ssh_connection_test.rs @@ -3,7 +3,9 @@ use poem_openapi::{ApiResponse, Object, OpenApi}; use russh::keys::PublicKeyBase64; use uuid::Uuid; use warpgate_common::{AdminPermission, WarpgateError}; -use warpgate_protocol_ssh::{RCCommand, RCEvent, RemoteClient, resolve_ssh_chain}; +use warpgate_protocol_ssh::{ + ConnectionError, IdentityHint, RCCommand, RCEvent, RemoteClient, resolve_ssh_chain, +}; use super::AdminContext; @@ -45,36 +47,49 @@ impl Api { let ssh_chain = resolve_ssh_chain(admin.services(), body.target_id, admin.auth.username()).await?; - let Some(target) = ssh_chain.last() else { - return Err(WarpgateError::InconsistentState( - "Did not resolve SSH chain".into(), - )); - }; - let (target_host, target_port) = (target.ssh_options.host.clone(), target.ssh_options.port); - - let ssh_chain = ssh_chain - .into_iter() - .map(|x| x.ssh_options) - .collect::>(); - let mut handles = RemoteClient::create(Uuid::new_v4(), admin.services().clone())?; - let _ = handles - .command_tx - .send((RCCommand::Connect(ssh_chain), None)); + // Not `Connect`: that would carry on into authenticating to the target + // once the key had been reported, opening a session nothing is attached + // to — and, for a certificate target, minting a real certificate to do + // it with. + let _ = handles.command_tx.send(( + RCCommand::CheckHostKey { + chain: ssh_chain, + // By identity. Which hop answers is decided by which target was + // asked about, not by which happens to be last. + target_id: body.target_id, + requested_by: if admin.auth.attribution_is_gateway() { + IdentityHint::Gateway(admin.auth.attribution().to_owned()) + } else { + IdentityHint::Person(admin.auth.attribution().to_owned()) + }, + }, + None, + )); + + // Kept out of the future below so the connection can be torn down after + // it resolves. `CheckHostKey` already ends the client task on its own; + // this is the second lock on the same door, scoped to this caller so + // that a real session's graceful disconnect stays untouched. + let abort_tx = handles.abort_tx.clone(); let fut = async move { let key = loop { match handles.event_rx.recv().await { - Some(RCEvent::HostKeyReceived(key, host, port)) - if host == target_host && port == target_port => - { - break key; - } - Some(RCEvent::HostKeyUnknown(_, host, port, reply)) => { - // this is a jump host, target key would hit HostKeyReceived - let _ = reply.send(false); - anyhow::bail!("Jump host {host}:{port} has an untrusted host key"); - } + // The address rides along with the key since #2437, and is + // deliberately not matched on here. Upstream needed it + // because their walk reported every hop; ours reports only + // the hop the caller named, decided by identity in + // `connect_chain`. Filtering by address on top of that is + // not a second opinion — both values come off the same + // resolved hop — and it made two integration tests pass + // with that identity gate disabled, which is the one thing + // they exist to notice. + // + // An untrusted jump host never reaches this loop as + // `HostKeyUnknown` either: the walk refuses it at the hop + // and arrives here as `ConnectionError::UntrustedJumpHost`. + Some(RCEvent::HostKeyReceived(key, _, _)) => break key, Some(RCEvent::ConnectionError(err)) => return Err(anyhow::Error::from(err)), Some(RCEvent::Error(err)) => return Err(err), None => anyhow::bail!("Failed to connect to target"), @@ -84,18 +99,24 @@ impl Api { anyhow::Ok(key) }; - // Result is matched manually since we need to manually format - // the error message with :# to included the nested errors here - match fut.await { + let result = fut.await; + let _ = abort_tx.send(()); + + // Sanitised, like every other place a connection error is shown to a + // person. Reaching a target behind a jump host authenticates that hop, + // so a Vault failure is reachable here and used to be rendered with its + // whole source chain — mount, policy and all. + match result { Ok(key) => Ok(CheckSshHostKeyResponse::Ok(Json( CheckSshHostKeyResponseBody { remote_key_type: key.algorithm().as_str().into(), remote_key_base64: key.public_key_base64(), }, ))), - Err(err) => Ok(CheckSshHostKeyResponse::Error(PlainText(format!( - "{err:#}" - )))), + Err(err) => Ok(CheckSshHostKeyResponse::Error(PlainText( + err.downcast_ref::() + .map_or_else(|| format!("{err:#}"), ConnectionError::client_message), + ))), } } } diff --git a/warpgate-admin/src/api/targets.rs b/warpgate-admin/src/api/targets.rs index 4bb10c713..074875620 100644 --- a/warpgate-admin/src/api/targets.rs +++ b/warpgate-admin/src/api/targets.rs @@ -9,8 +9,8 @@ use sea_orm::{ use uuid::Uuid; use warpgate_common::encryption::idempotent_maybe_encrypt_secret; use warpgate_common::{ - AdminPermission, Role as RoleConfig, Target as TargetConfig, TargetOptions, TargetSSHOptions, - WarpgateError, map_target_secrets, + AdminPermission, Role as RoleConfig, SSHTargetAuth, Target as TargetConfig, TargetOptions, + TargetSSHOptions, WarpgateError, map_target_secrets, }; use warpgate_db_entities::Target::TargetKind; use warpgate_db_entities::{KnownHost, Role, Target, TargetRoleAssignment, Ticket, TicketRequest}; @@ -65,6 +65,24 @@ enum CreateTargetResponse { BadRequest(Json), } +/// Whether a target's options name a Vault role the signing path could use. +/// +/// Checked when the target is saved, not only when a session tries to use it. +/// The connect path validates the role and refuses — correctly, and hours later, +/// with an error naming the session rather than the form that accepted the typo. +fn vault_role_is_usable(options: &TargetOptions) -> bool { + let TargetOptions::Ssh(ssh) = options else { + return true; + }; + let SSHTargetAuth::Certificate(certificate) = &ssh.auth else { + return true; + }; + certificate + .role + .as_ref() + .is_none_or(|role| warpgate_common::vault_name_is_well_formed(role)) +} + pub struct ListApi; #[OpenApi] @@ -127,6 +145,10 @@ impl ListApi { return Ok(CreateTargetResponse::BadRequest(Json("name".into()))); } + if !vault_role_is_usable(&body.options) { + return Ok(CreateTargetResponse::BadRequest(Json("role".into()))); + } + let db = &admin.services().db; let existing = Target::Entity::find() .filter(Target::Column::Name.eq(body.name.clone())) @@ -235,7 +257,7 @@ impl DetailApi { return Ok(UpdateTargetResponse::NotFound); }; - if target.kind != (&body.options).into() { + if target.kind != (&body.options).into() || !vault_role_is_usable(&body.options) { return Ok(UpdateTargetResponse::BadRequest); } @@ -472,3 +494,57 @@ impl RolesApi { Ok(DeleteTargetRoleResponse::Deleted) } } + +#[cfg(test)] +mod tests { + use warpgate_common::{ + SSHTargetAuth, SshTargetCertificateAuth, SshTargetPublicKeyAuth, TargetOptions, + TargetSSHOptions, + }; + + use super::vault_role_is_usable; + + fn ssh_target(auth: SSHTargetAuth) -> TargetOptions { + TargetOptions::Ssh(TargetSSHOptions { + host: "localhost".to_owned(), + port: 22, + username: "root".to_owned(), + allow_insecure_algos: None, + auth, + jump_host: None, + }) + } + + fn certificate_target(role: Option<&str>) -> TargetOptions { + ssh_target(SSHTargetAuth::Certificate(SshTargetCertificateAuth { + role: role.map(str::to_owned), + ..Default::default() + })) + } + + /// The admin API used to take a role the signing path would refuse, so the + /// operator learned of the typo from a broken session rather than from the + /// form that accepted it. + #[test] + fn a_role_the_signing_path_would_refuse_is_refused_at_save_time() { + assert!(vault_role_is_usable(&certificate_target(Some("warpgate")))); + assert!(vault_role_is_usable(&certificate_target(None))); + assert!( + !vault_role_is_usable(&certificate_target(Some("warp/gate"))), + "a role with a path separator was accepted at save time" + ); + assert!( + !vault_role_is_usable(&certificate_target(Some(""))), + "an empty role was accepted at save time" + ); + } + + /// A target with no Vault role has nothing to check, and must not be + /// refused for lacking one. + #[test] + fn a_target_without_a_vault_role_is_left_alone() { + assert!(vault_role_is_usable(&ssh_target(SSHTargetAuth::PublicKey( + SshTargetPublicKeyAuth::default() + )))); + } +} diff --git a/warpgate-admin/src/api/users.rs b/warpgate-admin/src/api/users.rs index 29f4b9c0e..b8b5f39bf 100644 --- a/warpgate-admin/src/api/users.rs +++ b/warpgate-admin/src/api/users.rs @@ -11,12 +11,30 @@ use warpgate_common::{ AdminPermission, AdminRole as AdminRoleConfig, User as UserConfig, UserRequireCredentialsPolicy, WarpgateError, }; +use warpgate_common_http::auth::TOKEN_ATTRIBUTIONS; use warpgate_core::logging::{AuditEvent, format_related_ids}; use warpgate_db_entities::{AdminRole, Role, User, UserAdminRoleAssignment, UserRoleAssignment}; use super::AdminContext; use crate::api::common::case_insensitive_search; +/// A username may not contain `:`. +/// +/// It is echoed into the certificate key ID as `warpgate::`, +/// which the target's own sshd log carries and which anything reading that log +/// splits on the colon. A name with one in it silently shifts every field, so +/// the person a session is attributed to is not the person who opened it — +/// which is the single claim the certificate feature makes. +/// The names the API tokens are attributed under are also refused, for the +/// same reason one field further along: `attribution()` puts `admin-token` into +/// the key ID when the admin API token drives a session, and nothing stopped an +/// admin creating a user by that name. Two sessions with different actors then +/// read identically in the target's log and in Vault's, which is the pair of +/// records this feature exists to make trustworthy. +fn username_is_well_formed(username: &str) -> bool { + !username.is_empty() && !username.contains(':') && !TOKEN_ATTRIBUTIONS.contains(&username) +} + #[derive(Object)] struct CreateUserRequest { username: String, @@ -84,7 +102,7 @@ impl ListApi { ) -> Result { admin.require(AdminPermission::UsersCreate)?; - if body.username.is_empty() { + if !username_is_well_formed(&body.username) { return Ok(CreateUserResponse::BadRequest(Json("name".into()))); } @@ -240,7 +258,7 @@ impl DetailApi { return Ok(UpdateUserResponse::NotFound); }; - if body.username.is_empty() { + if !username_is_well_formed(&body.username) { return Ok(UpdateUserResponse::BadRequest(Json("username".into()))); } @@ -869,3 +887,36 @@ impl RolesApi { Ok(DeleteUserAdminRoleResponse::Deleted) } } + +#[cfg(test)] +mod tests { + use super::username_is_well_formed; + + /// The `:` rejection had no test in either language, and no matrix guard — + /// the only check in this feature with neither. It protects the one claim + /// the certificate path makes: that `warpgate::` in the + /// target's sshd log names the person who opened the session. + #[test] + fn a_username_with_a_colon_would_shift_every_field_of_the_key_id() { + // What the target's sshd log carries, and what reads it back. + let key_id = |username: &str| format!("warpgate:{username}:0e5f"); + let field = |id: &str, n: usize| id.split(':').nth(n).unwrap_or_default().to_owned(); + + assert!(username_is_well_formed("alice")); + assert_eq!(field(&key_id("alice"), 1), "alice"); + + // Without the check: the reader attributes the session to `root`, and + // an operator auditing the target's logs sees a name that was chosen + // rather than authenticated. + assert_eq!(field(&key_id("root:admin"), 1), "root"); + assert!(!username_is_well_formed("root:admin")); + + // A token is not a person, and a person may not take a token's name. + assert!(!username_is_well_formed("admin-token")); + assert!(!username_is_well_formed("cluster-token")); + + assert!(!username_is_well_formed(":")); + assert!(!username_is_well_formed("trailing:")); + assert!(!username_is_well_formed("")); + } +} diff --git a/warpgate-aws/src/error.rs b/warpgate-aws/src/error.rs index fdac26d2a..03e9e014a 100644 --- a/warpgate-aws/src/error.rs +++ b/warpgate-aws/src/error.rs @@ -19,8 +19,13 @@ pub enum AwsError { ResourceNotFound(AwsResourceType, String), #[error("no AWS credentials available")] NoCredentials, + #[error( + "static AWS credentials (AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY without a session token) are disallowed; Vault authentication requires a temporary workload identity (instance profile, pod identity, or IRSA)" + )] + StaticCredentialsDisallowed, #[error("credentials: {0}")] Credentials(#[from] CredentialsError), + #[error("signing parameters: {0}")] SigningParams(#[from] BuildError), #[error("signing: {0}")] @@ -35,4 +40,17 @@ impl AwsError { pub fn sdk_error(err: E) -> Self { Self::Other(Box::new(err)) } + + pub fn client_message(&self) -> &'static str { + match self { + AwsError::NoCredentials + | AwsError::StaticCredentialsDisallowed + | AwsError::Credentials(_) => "AWS credential provider error", + AwsError::SigningParams(_) | AwsError::Signing(_) | AwsError::Http(_) => { + "AWS signing request failed" + } + AwsError::RegionUnknown(_) | AwsError::ResourceNotFound(_, _) => "AWS resource error", + AwsError::Other(_) => "AWS integration error", + } + } } diff --git a/warpgate-aws/src/lib.rs b/warpgate-aws/src/lib.rs index 96c4c8d37..2c6cac5d4 100644 --- a/warpgate-aws/src/lib.rs +++ b/warpgate-aws/src/lib.rs @@ -9,6 +9,7 @@ mod error; mod rds; mod region; mod s3; +mod sts_identity; pub use ec2::{Ec2InstanceInfo, find_instance_by_ip, is_running_on_ec2, send_ssh_public_key}; pub use eks::{EksClusterInfo, find_eks_cluster_by_url, generate_eks_token}; @@ -19,6 +20,7 @@ pub use s3::{ AutoCredentials, S3Credentials, S3MultipartUpload, S3Storage, S3StorageConfig, StaticCredentials, }; +pub use sts_identity::{StsIdentityRequest, sign_sts_identity_request}; /// Cached EC2 detection result static EC2_DETECTION: OnceCell = OnceCell::const_new(); diff --git a/warpgate-aws/src/sts_identity.rs b/warpgate-aws/src/sts_identity.rs new file mode 100644 index 000000000..504cf2ba0 --- /dev/null +++ b/warpgate-aws/src/sts_identity.rs @@ -0,0 +1,200 @@ +use std::collections::HashMap; +use std::time::SystemTime; + +use aws_credential_types::provider::ProvideCredentials; +use aws_sigv4::http_request::{SignableBody, SignableRequest, SigningSettings, sign}; +use aws_sigv4::sign::v4; +use tracing::debug; + +use crate::error::AwsError; + +const STS_BODY: &str = "Action=GetCallerIdentity&Version=2011-06-15"; + +/// A signed `sts:GetCallerIdentity` request, in the four parts a verifier needs +/// to replay it against STS and learn who signed it. +/// +/// The signature is what proves identity — no credential is disclosed — which is +/// why this can be handed to a third party such as Vault's AWS auth method. +pub struct StsIdentityRequest { + pub method: &'static str, + pub url: String, + pub body: String, + pub headers: HashMap, +} + +/// Names, never values. +/// +/// `headers` carries `authorization` and `x-amz-security-token`: the SigV4 +/// signature and the session token itself. A derived `Debug` put both in plain +/// text anywhere this was logged, traced or included in another type's derived +/// `Debug` — and the whole point of this crate is that the credential is not +/// disclosed to the party being handed the request. +impl std::fmt::Debug for StsIdentityRequest { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let mut names: Vec<&str> = self.headers.keys().map(String::as_str).collect(); + names.sort_unstable(); + f.debug_struct("StsIdentityRequest") + .field("method", &self.method) + .field("url", &self.url) + .field("body", &self.body) + .field("headers", &names) + .finish() + } +} + +/// Signs a `GetCallerIdentity` call with whatever the default credential chain +/// provides — on EC2 that is the instance role, whose credentials are short-lived +/// and never touch disk. Static access keys work too, but reintroduce exactly the +/// long-lived secret this authentication method exists to avoid. +/// +/// `server_id` is bound into the signature as the `X-Vault-AWS-IAM-Server-ID` +/// header when set, so a signed request captured by one verifier cannot be +/// replayed against another. +/// +/// `region` selects the regional STS endpoint. Leave it unset unless the verifier +/// is configured for one specific region: a signature scoped to a region is +/// rejected when the verifier replays it against the global endpoint, which is +/// what Vault does by default. +pub async fn sign_sts_identity_request( + region: Option<&str>, + server_id: Option<&str>, +) -> Result { + // The global endpoint expects signatures scoped to us-east-1. + let (host, signing_region) = match region { + Some(region) => (format!("sts.{region}.amazonaws.com"), region), + None => ("sts.amazonaws.com".to_owned(), "us-east-1"), + }; + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .region(aws_sdk_sts::config::Region::new(signing_region.to_string())) + .load() + .await; + + let credentials = config + .credentials_provider() + .ok_or(AwsError::NoCredentials)? + .provide_credentials() + .await?; + + if credentials.session_token().is_none() { + return Err(AwsError::StaticCredentialsDisallowed); + } + + let identity = credentials.into(); + + let url = format!("https://{host}/"); + + let mut headers = vec![ + ( + "content-type", + "application/x-www-form-urlencoded".to_owned(), + ), + ("host", host.clone()), + ]; + if let Some(server_id) = server_id { + headers.push(("x-vault-aws-iam-server-id", server_id.to_owned())); + } + + let signing_params = v4::SigningParams::builder() + .identity(&identity) + .region(signing_region) + .name("sts") + .time(SystemTime::now()) + .settings(SigningSettings::default()) + .build()?; + + let signable_request = SignableRequest::new( + "POST", + &url, + headers.iter().map(|(name, value)| (*name, value.as_str())), + SignableBody::Bytes(STS_BODY.as_bytes()), + )?; + + let (signing_instructions, _signature) = + sign(signable_request, &signing_params.into())?.into_parts(); + + let mut request = http::Request::builder().method("POST").uri(&url); + for (name, value) in &headers { + request = request.header(*name, value); + } + let mut request = request.body(())?; + signing_instructions.apply_to_request_http1x(&mut request); + + let headers = request + .headers() + .iter() + .filter_map(|(name, value)| { + Some((name.as_str().to_owned(), value.to_str().ok()?.to_owned())) + }) + .collect(); + + // What is left behind here, stated exactly rather than implied by the + // zeroization the rest of this path does have. + // + // Three values hold the session token and none of them can be wiped from + // here: the `Credentials` from the provider, the `identity` it is consumed + // into, and the signed request's header map. The first two are `Arc`-backed + // AWS SDK types with no `Drop` that clears them and no way to reach their + // buffers; `http::HeaderValue` exposes no mutable bytes. Clearing any of + // them needs `unsafe`, which this workspace denies. They are freed holding + // their contents. + // + // What that is worth: these are STS session credentials with an hour or so + // to live, not the long-lived access key this auth method exists to avoid — + // `StaticCredentialsDisallowed` above refuses that outright. The map + // returned below *is* wiped, by the caller in `warpgate-vault`, so the copy + // that travels furthest is the one that is handled. + // + // Say it here rather than fixing it silently: an earlier round recorded + // this as closed on the strength of a fix to a different copy. + drop(identity); + drop(request); + + debug!(signing_region, "Signed an STS GetCallerIdentity request"); + + Ok(StsIdentityRequest { + method: "POST", + url, + body: STS_BODY.to_owned(), + headers, + }) +} + +#[cfg(test)] +mod tests { + use super::StsIdentityRequest; + + /// The reason `Debug` is written by hand. A derived one printed the + /// signature and the session token. + #[test] + fn debug_shows_header_names_but_never_their_values() { + let request = StsIdentityRequest { + method: "POST", + url: "https://sts.amazonaws.com/".to_owned(), + body: super::STS_BODY.to_owned(), + headers: [ + ( + "authorization".to_owned(), + "AWS4-HMAC-SHA256 SECRET".to_owned(), + ), + ( + "x-amz-security-token".to_owned(), + "SESSION-TOKEN".to_owned(), + ), + ] + .into_iter() + .collect(), + }; + + let rendered = format!("{request:?}"); + assert!(rendered.contains("authorization"), "{rendered}"); + assert!(rendered.contains("x-amz-security-token"), "{rendered}"); + assert!( + !rendered.contains("SECRET"), + "the signature is in {rendered}" + ); + assert!( + !rendered.contains("SESSION-TOKEN"), + "the token is in {rendered}" + ); + } +} diff --git a/warpgate-common-http/src/auth.rs b/warpgate-common-http/src/auth.rs index 3fd76e855..892f53d98 100644 --- a/warpgate-common-http/src/auth.rs +++ b/warpgate-common-http/src/auth.rs @@ -218,6 +218,17 @@ impl FullUserAuthorization { } } +/// The names `attribution()` gives to the two tokens, and the names no user may +/// have. +/// +/// Exported so the admin API can refuse them: `attribution()` returns these +/// into the certificate key ID, which the target's sshd log carries verbatim, +/// and nothing stopped an admin creating a user called `admin-token`. A session +/// opened by that person and one opened by the admin API token then read +/// identically in the target's log and in Vault's audit log — the two records +/// this feature exists to make trustworthy. +pub const TOKEN_ATTRIBUTIONS: [&str; 2] = ["admin-token", "cluster-token"]; + impl RequestAuthorization { /// Returns a username if one is present (admin token has none) pub const fn username(&self) -> Option<&String> { @@ -228,6 +239,35 @@ impl RequestAuthorization { } } + /// A name for a log that is not ours, honest for every variant. + /// + /// `username()` returns `None` for a token, and the one caller that needed + /// a name substituted the literal string "admin" — so a certificate minted + /// by an API token was recorded, in the target's own sshd log and in + /// Vault's issuance log, as though a person called "admin" had opened the + /// session. That is the attribution failure the certificate feature exists + /// to prevent, reintroduced by the change that was meant to fix it. + /// + /// A token is not a person and this says so. + pub fn attribution(&self) -> &str { + match self { + Self::Session(auth) => auth.username(), + Self::UserToken { username, .. } => username, + Self::AdminToken => TOKEN_ATTRIBUTIONS[0], + Self::ClusterToken => TOKEN_ATTRIBUTIONS[1], + } + } + + /// Whether `attribution()` names the gateway rather than a person. + /// + /// `username()` already draws this line — it is `None` for exactly the two + /// token variants. This asks a different question with the same answer: not + /// "who is the user" but "is this string ours, to be kept verbatim". + #[must_use] + pub const fn attribution_is_gateway(&self) -> bool { + matches!(self, Self::AdminToken | Self::ClusterToken) + } + /// Returns a user ID if present in the authorization context or nil UUID pub const fn user_id(&self) -> Uuid { match self { diff --git a/warpgate-common/src/config/defaults.rs b/warpgate-common/src/config/defaults.rs index 0c21c4eaf..a6f4b6b53 100644 --- a/warpgate-common/src/config/defaults.rs +++ b/warpgate-common/src/config/defaults.rs @@ -1,8 +1,33 @@ use std::net::{Ipv6Addr, SocketAddr}; +use std::path::PathBuf; use std::time::Duration; use crate::{ListenEndpoint, Secret}; +pub fn _default_vault_ssh_mount() -> String { + "ssh-client-signer".to_owned() +} + +pub const fn _default_vault_timeout() -> Duration { + Duration::from_secs(10) +} + +pub fn _default_vault_kubernetes_token_path() -> PathBuf { + PathBuf::from("/var/run/secrets/kubernetes.io/serviceaccount/token") +} + +pub fn _default_azure_resource() -> String { + "https://management.azure.com/".to_owned() +} + +pub fn _default_azure_imds() -> String { + "http://169.254.169.254".to_owned() +} + +pub fn _default_gcp_metadata() -> String { + "http://metadata.google.internal".to_owned() +} + pub const fn _default_true() -> bool { true } diff --git a/warpgate-common/src/config/mod.rs b/warpgate-common/src/config/mod.rs index e890b1ea1..cc5a9890e 100644 --- a/warpgate-common/src/config/mod.rs +++ b/warpgate-common/src/config/mod.rs @@ -7,11 +7,13 @@ use std::path::PathBuf; use std::time::Duration; use defaults::{ - _default_audit_retention, _default_cookie_max_age, _default_database_url, _default_false, + _default_audit_retention, _default_azure_imds, _default_azure_resource, + _default_cookie_max_age, _default_database_url, _default_false, _default_gcp_metadata, _default_http_listen, _default_kubernetes_listen, _default_mysql_advertised_version, _default_mysql_listen, _default_postgres_listen, _default_rdp_listen, _default_recordings_path, _default_retention, _default_session_max_age, _default_ssh_inactivity_timeout, - _default_ssh_keys_path, _default_ssh_listen, _default_vnc_listen, + _default_ssh_keys_path, _default_ssh_listen, _default_vault_kubernetes_token_path, + _default_vault_ssh_mount, _default_vault_timeout, _default_vnc_listen, }; use poem_openapi::{Object, Union}; use schemars::JsonSchema; @@ -410,6 +412,144 @@ pub enum LogFormat { Json, } +/// How Warpgate proves its own identity to Vault. +/// +/// Both methods read their credential from a file rather than from the config, +/// deliberately: the point of issuing target credentials on demand is that +/// nothing long-lived sits on the Warpgate host, and a value pasted into the +/// config would put it straight back. A Kubernetes service account token is +/// mounted and rotated by the kubelet; an AppRole secret ID is short-lived and +/// meant to be delivered response-wrapped by whatever provisions the host. +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, JsonSchema)] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum VaultAuth { + Kubernetes { + role: String, + #[serde(default = "_default_vault_kubernetes_token_path")] + token_path: PathBuf, + }, + AppRole { + // Vault treats this as public — half a credential, useless without the + // secret ID. Wrapped anyway, so that every field here that is part of + // an authentication is redacted by one rule rather than by a judgement + // about which halves matter. A `///` comment would put this rationale + // in `config-schema.json` as operator-facing documentation, which it is + // not. + #[schemars(with = "String")] + role_id: Secret, + secret_id_path: PathBuf, + }, + /// Signs an `sts:GetCallerIdentity` call with the default credential chain — + /// on EC2 that is the instance role. Access keys in the environment work but + /// put back the long-lived secret this exists to avoid. + Aws { + #[serde(default)] + role: Option, + /// Bound into the signature so a captured request cannot be replayed + /// against a different Vault. Must match the server's `iam_server_id_header_value`. + #[serde(default)] + server_id: Option, + /// Signs against a regional STS endpoint instead of the global one. Set + /// this only when Vault is configured with a matching `sts_endpoint`: + /// Vault replays the request globally by default, and a region-scoped + /// signature is rejected there. + #[serde(default)] + region: Option, + }, + Azure { + role: String, + #[serde(default = "_default_azure_resource")] + resource: String, + #[serde(default = "_default_azure_imds")] + metadata_address: String, + }, + Gcp { + role: String, + #[serde(default = "_default_gcp_metadata")] + metadata_address: String, + }, +} + +impl VaultAuth { + pub const fn kind(&self) -> &'static str { + match self { + Self::Kubernetes { .. } => "kubernetes", + Self::AppRole { .. } => "approle", + Self::Aws { .. } => "aws", + Self::Azure { .. } => "azure", + Self::Gcp { .. } => "gcp", + } + } +} + +/// The longest a session certificate may be valid for. +/// +/// Generous — a role would have to be badly misconfigured to exceed it — but the +/// point of this feature is a credential that is worthless a few minutes after +/// it is issued, and nothing else anywhere checks that. +/// +/// Here rather than beside either user, because it is checked at both ends and +/// they are in different crates: `certificate_ttl` is refused against it when +/// the config loads, and a certificate that comes back exceeding it is refused +/// when it arrives. Two copies of one number is how they drift apart. +pub const MAX_CERTIFICATE_LIFETIME: Duration = Duration::from_secs(24 * 60 * 60); + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, JsonSchema)] +pub struct VaultConfig { + /// Base URL of the Vault server, e.g. `https://vault.internal:8200`. + pub address: String, + + /// Mount point of the SSH secrets engine in signed-certificates mode. + #[serde(default = "_default_vault_ssh_mount")] + pub mount: String, + + /// Signing role used by targets that don't name one of their own. + pub default_role: String, + + pub auth: VaultAuth, + + /// Lifetime asked for when signing. Vault clamps this to the role's + /// `max_ttl`, so it can only shorten what the role already allows — set it + /// to hold the window down without editing Vault. Left unset, the role's own + /// TTL decides. + #[serde(default, with = "humantime_serde::option")] + #[schemars(with = "Option")] + pub certificate_ttl: Option, + + #[serde(default = "_default_vault_timeout", with = "humantime_serde")] + #[schemars(with = "String")] + pub timeout: Duration, + + /// PEM file holding the CA that issued Vault's certificate, for a Vault + /// behind a private CA. + /// + /// Added to the host's trust store rather than replacing it, so a + /// misconfigured path cannot silently turn verification off — an + /// unreadable or malformed file is a startup error. There is deliberately + /// no switch to skip verification: the Vault token crosses this connection + /// in a header, and unlike the HTTP and Kubernetes target paths, which + /// offer `verify: false` for devices whose certificates cannot be fixed, + /// there is no equivalent case here. + #[serde(default)] + pub ca_bundle: Option, + + /// The signing CA the target trusts, in OpenSSH public-key format + /// (`ssh-ed25519 AAAA…`), pinned so a certificate signed by anything else + /// is refused before it is offered. + /// + /// Every other check on Vault's response asks whether the certificate is + /// what was requested. This is the only one that asks *who signed it*. The + /// target's `TrustedUserCAKeys` is the real enforcement and would refuse + /// such a certificate anyway — but it does so after Warpgate has offered + /// it, and the refusal that comes back names the target rather than the + /// issuer that mis-signed. Pinning turns a confusing rejection into a + /// precise one, and detects a role rebound to a different CA. + /// + /// Left unset, nothing is checked here. + #[serde(default)] + pub ca_public_key: Option, +} + #[derive(Debug, Deserialize, Serialize, Clone, JsonSchema)] pub struct SshConfig { #[serde(default = "_default_false")] @@ -883,6 +1023,10 @@ pub struct WarpgateConfigStore { #[schemars(with = "String")] pub database_url: Secret, + /// Absent unless the deployment issues target credentials from Vault. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub vault: Option, + #[serde(default)] pub ssh: SshConfig, @@ -915,6 +1059,7 @@ impl Default for WarpgateConfigStore { recordings: <_>::default(), external_host: None, database_url: _default_database_url(), + vault: None, ssh: <_>::default(), http: <_>::default(), kubernetes: <_>::default(), diff --git a/warpgate-common/src/config/target.rs b/warpgate-common/src/config/target.rs index 55efd645e..dccf6356f 100644 --- a/warpgate-common/src/config/target.rs +++ b/warpgate-common/src/config/target.rs @@ -51,6 +51,8 @@ pub enum SSHTargetAuth { Password(SshTargetPasswordAuth), #[serde(rename = "publickey")] PublicKey(SshTargetPublicKeyAuth), + #[serde(rename = "certificate")] + Certificate(SshTargetCertificateAuth), #[serde(rename = "iam_role")] IamRole(SshTargetIamRoleAuth), } @@ -68,6 +70,80 @@ pub struct SshTargetPublicKeyAuth { pub key_id: Option, } +/// Whether a Vault mount or role name is one Vault can address. +/// +/// The rule lives here rather than in `warpgate-vault` because two crates need +/// it and only one of them can hold a Vault client: the admin API accepts a +/// role at save time, and used to accept one the signing path would reject at +/// connect time — an operator learning of the typo from a broken session +/// rather than from the form that took it. +#[must_use] +pub fn vault_name_is_well_formed(name: &str) -> bool { + !name.is_empty() + && name + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '_' || c == '-') +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Object, Default)] +pub struct SshTargetCertificateAuth { + /// Vault signing role for this target; `None` uses the configured default + /// role. The role is what constrains which principals may be requested, so + /// targets of differing privilege belong to differing roles. + #[serde(default)] + pub role: Option, + + /// Critical options this target's certificates may carry. + /// + /// Anything not listed here is refused, because the target's sshd enforces + /// whatever arrives: a `force-command` decides what the session runs, under + /// the connecting user's own principal and key ID. + /// + /// Pinning a `value` also makes the option **mandatory** — a certificate + /// without it is refused. Otherwise someone who can write the Vault role + /// but not sign with it removes the pinned `force-command` instead of + /// adding an option, and a target locked to one command hands out a shell. + /// A bare name only permits, which is how a role that sometimes sets an + /// option is expressed. + #[serde(default)] + #[oai(default)] + pub allowed_critical_options: Vec, + + /// Certificate extensions this target's certificates may carry. + /// + /// A separate authorization mechanism from critical options, and the one + /// that decides what a session can *do* rather than what it runs. OpenSSH + /// opens `direct-tcpip` only for `permit-port-forwarding` and reaches the + /// connecting user's own SSH agent only for `permit-agent-forwarding`, both + /// judged purely on what the certificate carries. So a pinned + /// `force-command` does not confine a session on its own: it governs the + /// shell and exec channels and nothing else. + /// + /// Defaults to `permit-pty` alone — enough for an interactive session and + /// nothing more. Anything else has to be named here, because the alternative + /// is that a Vault role's `default_extensions`, set deliberately or written + /// by someone with role-write and no signing right, silently grants + /// forwarding on a target that was supposed to be locked down. + #[serde(default = "_default_allowed_extensions")] + #[oai(default = "_default_allowed_extensions")] + pub allowed_extensions: Vec, +} + +fn _default_allowed_extensions() -> Vec { + vec!["permit-pty".to_owned()] +} + +#[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Object)] +pub struct SshCertificateCriticalOption { + /// Option name as it appears in the certificate, e.g. `force-command`. + pub name: String, + + /// The exact value required. Unset accepts any value for this name — worth + /// avoiding for `force-command`, whose value is the command that runs. + #[serde(default)] + pub value: Option, +} + #[derive(Debug, Deserialize, Serialize, Clone, PartialEq, Eq, Object, Default)] pub struct SshTargetIamRoleAuth {} diff --git a/warpgate-common/src/encryption.rs b/warpgate-common/src/encryption.rs index f4c1c224f..1075f78d6 100644 --- a/warpgate-common/src/encryption.rs +++ b/warpgate-common/src/encryption.rs @@ -406,10 +406,13 @@ mod tests { let (head, payload) = encrypted.rsplit_once(':').unwrap(); let mut bytes = BASE64.decode(payload.as_bytes()).unwrap(); - let last = bytes.len() - 1; - bytes.swap(0, last); + // Bitwise flip instead of swapping 0 and last bytes: random 12-byte GCM nonces + // can occasionally yield bytes[0] == bytes[last], making swap a no-op and causing + // false test failure. + bytes[0] ^= 0xff; let tampered = format!("{head}:{}", BASE64.encode(&bytes)); + assert!(matches!( k.decrypt(&tampered), Err(EncryptionError::Corrupt) diff --git a/warpgate-core/Cargo.toml b/warpgate-core/Cargo.toml index 85346131a..138cf8865 100644 --- a/warpgate-core/Cargo.toml +++ b/warpgate-core/Cargo.toml @@ -52,6 +52,7 @@ warpgate-db-migrations = { path = "../warpgate-db-migrations" } warpgate-ldap = { path = "../warpgate-ldap" } warpgate-sso = { path = "../warpgate-sso", default-features = false } warpgate-tls = { path = "../warpgate-tls" } +warpgate-vault = { path = "../warpgate-vault" } webpki = { version = "0.22", default-features = false } zune-jpeg.workspace = true diff --git a/warpgate-core/src/lib.rs b/warpgate-core/src/lib.rs index a6486d37e..3b18c8da8 100644 --- a/warpgate-core/src/lib.rs +++ b/warpgate-core/src/lib.rs @@ -19,6 +19,7 @@ pub mod recordings; mod services; mod state; pub mod ticket_requests; +mod vault_cell; pub use auth_state_store::*; pub use config_providers::*; pub use credential_encryption::*; @@ -28,3 +29,4 @@ pub use listener_status::*; pub use protocols::*; pub use services::*; pub use state::{SessionState, SessionStateInit, State}; +pub use vault_cell::VaultCell; diff --git a/warpgate-core/src/services.rs b/warpgate-core/src/services.rs index e61fa04c4..fcf2cf985 100644 --- a/warpgate-core/src/services.rs +++ b/warpgate-core/src/services.rs @@ -10,6 +10,7 @@ use tracing::warn; use warpgate_common::auth::{AuthState, CredentialKind}; use warpgate_common::{GlobalParams, Protocol, Secret, SessionId, WarpgateConfig, WarpgateError}; use warpgate_db_entities::Parameters; +use warpgate_vault::VaultClient; use crate::cluster::Cluster; use crate::db::connect_to_db_and_migrate; @@ -18,6 +19,7 @@ use crate::rate_limiting::RateLimiterRegistry; use crate::recordings::SessionRecordings; use crate::{ AuthStateStore, ConfigProviderEnum, DatabaseConfigProvider, ListenerStatusRegistry, State, + VaultCell, }; #[derive(Clone)] @@ -28,6 +30,9 @@ pub struct Services { pub cluster: Arc, pub state: Arc>, pub config_provider: Arc, + /// Empty unless the config declares a Vault server. Swappable so that + /// editing `vault:` takes effect without a restart. + pub vault: VaultCell, pub auth_state_store: Arc>, pub admin_token: Arc>>, pub cluster_token: Arc>, @@ -67,6 +72,31 @@ impl Services { config: WarpgateConfig, admin_token: Option, params: GlobalParams, + ) -> Result { + Self::build(config, admin_token, params, true).await + } + + /// For commands that never reach a target, and so never need Vault. + /// + /// `recover-access` is the break-glass path: it exists for the moment + /// something is already wrong. Building the Vault client eagerly made an + /// unusable `vault:` section — a namespaced mount like `team-a/ssh`, which + /// `validate_segment` rejects, or an unreadable `ca_bundle` — enough to stop + /// an administrator recovering their own access. Nothing in that command, + /// or in `create-user`, opens a session. + pub async fn new_without_vault( + config: WarpgateConfig, + admin_token: Option, + params: GlobalParams, + ) -> Result { + Self::build(config, admin_token, params, false).await + } + + async fn build( + config: WarpgateConfig, + admin_token: Option, + params: GlobalParams, + with_vault: bool, ) -> Result { let db = connect_to_db_and_migrate(&config, ¶ms).await?; let recordings = SessionRecordings::new(db.clone(), ¶ms); @@ -74,6 +104,18 @@ impl Services { let cluster = Arc::new(Cluster::new(db.clone(), config.store.http.listen.port()).await?); + let vault = VaultCell::new(if with_vault { + config + .store + .vault + .clone() + .map(VaultClient::new) + .transpose()? + .map(Arc::new) + } else { + None + }); + let config = Arc::new(Mutex::new(config)); let config_provider = Arc::new(DatabaseConfigProvider::new(&db).into()); @@ -124,6 +166,7 @@ impl Services { cluster, rate_limiter_registry, config_provider, + vault, auth_state_store, admin_token: Arc::new(admin_token.map(Secret::new)), cluster_token: Arc::new(resolve_cluster_token(&db).await?), diff --git a/warpgate-core/src/vault_cell.rs b/warpgate-core/src/vault_cell.rs new file mode 100644 index 000000000..9fae8633d --- /dev/null +++ b/warpgate-core/src/vault_cell.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; + +use tokio::sync::watch; +use warpgate_vault::VaultClient; + +/// Houses a replaceable reference to the Vault client, so that editing the +/// `vault:` section takes effect the way editing any other section does. +/// +/// Built on a `watch` channel for the same reason the listener supervisors are: +/// a client is not something a session can hold across a reload, since the +/// address, the mount or the way Warpgate proves its identity may all have +/// changed. Readers take a fresh reference per use; each keeps working until it +/// is dropped, so a session already in flight is never cut short by a reload. +#[derive(Clone)] +pub struct VaultCell { + receiver: watch::Receiver>>, + sender: watch::Sender>>, +} + +impl VaultCell { + pub fn new(client: Option>) -> Self { + let (sender, receiver) = watch::channel(client); + Self { receiver, sender } + } + + /// The client as of right now, or `None` when no Vault server is configured. + pub fn get(&self) -> Option> { + self.receiver.borrow().clone() + } + + pub fn replace(&self, client: Option>) { + let _ = self.sender.send(client); + } +} diff --git a/warpgate-protocol-http/src/api/info.rs b/warpgate-protocol-http/src/api/info.rs index d7b261a30..c957a2d87 100644 --- a/warpgate-protocol-http/src/api/info.rs +++ b/warpgate-protocol-http/src/api/info.rs @@ -133,6 +133,9 @@ pub struct Info { setup_state: Option, admin_permissions: Option, running_on_ec2: Option, + /// Whether a Vault server is configured, so the admin UI can hide target + /// authentication options that would fail at connect time. + has_vault: Option, should_prompt_analytics: bool, /// Login banner, shown to unauthenticated visitors too. Empty when unset. banner: String, @@ -357,6 +360,9 @@ impl Api { } else { None }, + has_vault: auth_ctx + .is_some() + .then(|| ctx.services().vault.get().is_some()), should_prompt_analytics, banner: parameters.banner.clone(), show_session_menu: parameters.show_session_menu, diff --git a/warpgate-protocol-ssh/Cargo.toml b/warpgate-protocol-ssh/Cargo.toml index 955e0152e..c78f7735b 100644 --- a/warpgate-protocol-ssh/Cargo.toml +++ b/warpgate-protocol-ssh/Cargo.toml @@ -13,6 +13,7 @@ curve25519-dalek = { version = "5.0.0", default-features = false } # pin due to dialoguer.workspace = true ed25519-dalek = { version = "3.0.0", default-features = false } # pin due to build fail on x86 in 2.1 futures.workspace = true +humantime = "2" natord = { version = "1.0", default-features = false } ratatui = { version = "0.30", default-features = false, features = [ "crossterm", @@ -39,4 +40,5 @@ warpgate-common-http = { path = "../warpgate-common-http", default-features = fa warpgate-core = { path = "../warpgate-core", default-features = false } warpgate-db-entities = { path = "../warpgate-db-entities", default-features = false } warpgate-tls = { path = "../warpgate-tls" } +warpgate-vault = { path = "../warpgate-vault" } zeroize = { version = "^1.5", default-features = false } diff --git a/warpgate-protocol-ssh/src/client/error.rs b/warpgate-protocol-ssh/src/client/error.rs index d67f8f209..46f66350d 100644 --- a/warpgate-protocol-ssh/src/client/error.rs +++ b/warpgate-protocol-ssh/src/client/error.rs @@ -2,6 +2,20 @@ use std::error::Error; use warpgate_common::WarpgateError; +/// What a connected user may be shown for an error carried as `anyhow::Error`. +/// +/// `RCEvent::Error` is typed `anyhow::Error`, so the concrete error has to be +/// recovered before it can be sanitised. Anything that does not downcast falls +/// back to a constant — the safe direction, and the only one available when the +/// type is unknown. +#[must_use] +pub fn client_error_message(error: &anyhow::Error) -> &'static str { + error.downcast_ref::().map_or( + "Internal error in the target connection", + SshClientError::client_message, + ) +} + #[derive(thiserror::Error, Debug)] pub enum SshClientError { #[error("mpsc error")] @@ -18,4 +32,24 @@ impl SshClientError { pub fn other(err: E) -> Self { Self::Other(Box::new(err)) } + + /// What a connected user may be shown. + /// + /// The same job `ConnectionError::client_message` does, and it exists for + /// the same reason: `Warpgate` here is `#[error(transparent)]`, so + /// `WarpgateError`'s own `Display` passes straight through — a database + /// failure renders as `database error: {DbErr}` carrying SQL text, and an + /// encryption-key mismatch names the configured key fingerprints. + /// + /// This error reaches a terminal through `RCEvent::Error`, which wrote + /// `format!("Error: {e}")` directly and so went around the sanitiser + /// entirely. One hardened path and one unhardened path to the same sink is + /// not a boundary. + pub const fn client_message(&self) -> &'static str { + match self { + Self::MpscError => "Internal connection error", + Self::Russh(_) => "SSH protocol error", + Self::Warpgate(_) | Self::Other(_) => "Internal error in the target connection", + } + } } diff --git a/warpgate-protocol-ssh/src/client/mod.rs b/warpgate-protocol-ssh/src/client/mod.rs index 1dd27af8d..89b530ccc 100644 --- a/warpgate-protocol-ssh/src/client/mod.rs +++ b/warpgate-protocol-ssh/src/client/mod.rs @@ -4,8 +4,10 @@ mod error; mod handler; use std::borrow::Cow; use std::collections::{HashMap, HashSet}; +use std::future::Future; use std::io; use std::net::ToSocketAddrs; +use std::pin::Pin; use std::sync::Arc; use std::time::Duration; @@ -13,11 +15,12 @@ use anyhow::Result; use bytes::Bytes; use channel_direct_tcpip::DirectTCPIPChannel; use channel_session::SessionChannel; -pub use error::SshClientError; +pub use error::{SshClientError, client_error_message}; use futures::{FutureExt, pin_mut}; use handler::ClientHandler; use russh::client::{AuthResult, Handle, KeyboardInteractiveAuthResponse}; -use russh::keys::{PrivateKeyWithHashAlg, PublicKey}; +use russh::keys::ssh_key::certificate::CertType; +use russh::keys::{Algorithm, Certificate, PrivateKey, PrivateKeyWithHashAlg, PublicKey}; use russh::{MethodKind, Preferred, Sig, kex, mac}; use serde::Serialize; use tokio::sync::mpsc::{ @@ -28,7 +31,12 @@ use tokio::task::JoinHandle; use tracing::*; use uuid::Uuid; use warpgate_aws::AwsError; -use warpgate_common::{SSHTargetAuth, SessionId, TargetOptions, TargetSSHOptions, WarpgateError}; +use warpgate_common::helpers::rng::get_crypto_rng; +use warpgate_common::{ + MAX_CERTIFICATE_LIFETIME, SSHTargetAuth, SessionId, SshCertificateCriticalOption, + TargetOptions, TargetSSHOptions, WarpgateError, +}; +use warpgate_common_http::auth::TOKEN_ATTRIBUTIONS; use warpgate_core::{ConfigProvider, Services}; use self::handler::ClientHandlerEvent; @@ -36,6 +44,142 @@ use super::{ChannelOperation, DirectTCPIPParams}; use crate::client::handler::ClientHandlerError; use crate::{ForwardedStreamlocalParams, ForwardedTcpIpParams, load_client_keys}; +/// What a hop in the chain is to the caller. +/// +/// Every hop presents a host key, so "the host key" is ambiguous the moment a +/// jump host is involved — and the answer the caller wants is the one from the +/// hop they named, which is always the last. +#[derive(Clone, Copy, PartialEq, Eq, Debug)] +enum HopRole { + /// An ordinary connection: every hop's key is reported, because the user + /// may be prompted to trust any of them. + Connecting, + /// On the way to the hop being asked about. Its key is verified against + /// known hosts as usual, but not reported as an answer. + TraversedWhileChecking, + /// The hop the caller asked about. Report its key and go no further. + CheckedHost, +} + +/// Which role a hop plays, given which target was asked about. +/// +/// By identity, not by position. It used to take "is this the last hop", which +/// is the same answer for every chain built today and a different question: the +/// resolver knows which target was named and threw that away, leaving +/// correctness resting on the chain always happening to terminate there. +fn role(check_target: Option, hop_id: Uuid) -> HopRole { + match check_target { + None => HopRole::Connecting, + Some(asked_about) if asked_about == hop_id => HopRole::CheckedHost, + Some(_) => HopRole::TraversedWhileChecking, + } +} + +impl HopRole { + const fn reports_host_key(self) -> bool { + matches!(self, Self::Connecting | Self::CheckedHost) + } + + const fn stops_after_host_key(self) -> bool { + matches!(self, Self::CheckedHost) + } +} + +/// How long a target may take to finish the SSH handshake. Generous for any +/// real server on any real link, and the only thing standing between a target +/// that accepts a connection and then goes quiet and a gateway task held for as +/// long as it likes. +const HANDSHAKE_TIMEOUT: Duration = Duration::from_secs(30); + +/// How far out the handshake bound is pushed while a host-key question is +/// outstanding. +/// +/// Not disarmed — pushed out — so that nothing has to remember to arm it again. +/// A year is "not while a person is reading a fingerprint" expressed as a +/// number. +const fn while_a_host_key_answer_is_outstanding() -> Duration { + Duration::from_secs(365 * 24 * 60 * 60) +} + +/// And what it comes back to once the answer arrives: the remaining handshake +/// is the target's again, so it gets the target's bound. +/// +/// A separate function from the constant because this is the thing that can be +/// got wrong. The first version of this pause never ended — the deadline was +/// pushed out and nothing brought it back — and a target that answered with a +/// host key and then went quiet held the session, the ephemeral key and a live +/// certificate until russh's inactivity timeout. +const fn once_the_host_key_is_answered() -> Duration { + HANDSHAKE_TIMEOUT +} + +/// How long authentication may take, when nothing else decides. +const AUTHENTICATION_TIMEOUT: Duration = Duration::from_secs(30); + +/// How long the target itself may take to answer one USERAUTH request. +/// +/// Deliberately not derived from `authentication_budget`: that budget has to +/// cover the issuer as well, and for a certificate target it scales with +/// `vault.timeout`, which nothing clamps. The target's own answer is not slower +/// because Vault is, so it does not get Vault's allowance. +const TARGET_USERAUTH_TIMEOUT: Duration = Duration::from_secs(30); + +/// Bounds one USERAUTH round trip to the target. +/// +/// Every credential type goes through this, so a new authentication method +/// cannot arrive unbounded by forgetting — which is how this gap opened in the +/// first place. +async fn bounded_userauth( + what: impl Future>, +) -> Result { + bounded_userauth_within(TARGET_USERAUTH_TIMEOUT, what).await +} + +/// The bound above, with the duration passed in so a test can assert it fires +/// without waiting thirty seconds for it. +/// +/// The alternative was tokio's pausable clock, which needs its `test-util` +/// feature. Turning that on changes feature unification for every crate sharing +/// the build and forces a full rebuild — measured here at minutes, paid by every +/// CI run and every first build after a checkout. A parameter costs one line. +async fn bounded_userauth_within( + bound: Duration, + what: impl Future>, +) -> Result { + tokio::time::timeout(bound, what) + .await + .map_err(|_| ConnectionError::TargetAuthenticationTimeout)? + .map_err(ConnectionError::from) +} + +/// The worst case a certificate authentication can spend on Vault, counted in +/// operations the client bounds rather than in HTTP requests: a token, a sign, +/// then — on a `403` — a second token and a second sign. Four, with one for +/// margin. +/// +/// A metadata fetch does not add to this. It happens inside `login_body()`, +/// which `token()` wraps in a single `config.timeout` together with the login +/// itself, so a method fetching twice still spends one bounded operation +/// getting a token. Counting requests instead of bounds is what made this +/// constant look wrong when it was not. +const VAULT_CALLS_PER_AUTHENTICATION: u32 = 5; + +/// How long authentication may take, as distinct from the transport handshake. +/// +/// The two shared `HANDSHAKE_TIMEOUT` until now: different phases, different +/// causes, one constant by accident. A certificate target's authentication is +/// dominated by the issuer, and five calls at the default 10s `vault.timeout` +/// is fifty seconds against a thirty-second bound — so a Vault that was slow +/// but working produced a timeout naming the target, which is not the party +/// that was slow. +fn authentication_budget(auth: &SSHTargetAuth, vault_timeout: Option) -> Duration { + match (auth, vault_timeout) { + (SSHTargetAuth::Certificate(_), Some(per_call)) => AUTHENTICATION_TIMEOUT + .max(per_call * VAULT_CALLS_PER_AUTHENTICATION + Duration::from_secs(5)), + _ => AUTHENTICATION_TIMEOUT, + } +} + #[derive(Debug, thiserror::Error)] pub enum ConnectionError { #[error("Host key mismatch")] @@ -58,6 +202,9 @@ pub enum ConnectionError { #[error("AWS: {0}")] Aws(#[from] AwsError), + #[error("Vault: {0}")] + Vault(#[from] warpgate_vault::VaultError), + #[error("Could not resolve address")] Resolve, @@ -67,17 +214,530 @@ pub enum ConnectionError { #[error("Aborted")] Aborted, - #[error("Authentication failed")] - Authentication, + /// The peer accepted the connection and then did not finish the handshake. + /// Distinct from a refusal, because nothing was refused — and distinct from + /// an ordinary timeout, because it names the stage. + #[error("The SSH target did not complete the handshake in time")] + HandshakeTimeout, + + /// Carries why, because the target's own reason is the only thing that + /// distinguishes a wrong credential from a clock that disagrees — and a + /// message that reaches only the server log is not much use to whoever is + /// staring at a closed session. + #[error("Authentication failed: {0}")] + Authentication(String), + + /// Warpgate refused the credential before offering it, so the target never + /// saw anything — saying it was "rejected by the target" would name the + /// wrong party and send the operator to the wrong logs. + #[error("Certificate refused by Warpgate: {0}")] + CertificateRefused(String), #[error("Jump host target not found")] JumpHostTargetNotFound, + /// Only reachable while checking a target's host key: the chain leading to + /// it goes through a host that is not trusted yet. + #[error("A jump host on the way to this target has an untrusted host key")] + UntrustedJumpHost, + + /// Authentication did not finish in time. + /// + /// Its own variant rather than `HandshakeTimeout`, and its own budget: + /// for a certificate target this phase is dominated by the issuer, not by + /// the target, and naming the target sends whoever is debugging to the + /// wrong logs. + #[error("Authentication to the SSH target did not complete in time")] + AuthenticationTimeout, + + /// The target received a USERAUTH request and never answered it. + /// + /// Its own variant and its own bound, separate from `AuthenticationTimeout`. + /// That one sizes the whole step, and for a certificate target it grows with + /// `vault.timeout` — a value config does not clamp from above. Sharing it + /// meant a target that went quiet the moment it received its certificate + /// held the session, the ephemeral private key and a live certificate for a + /// window measured in the issuer's slowness rather than its own: 55 seconds + /// by default, and unbounded above. + #[error("The SSH target did not answer the authentication request in time")] + TargetAuthenticationTimeout, + + /// A jump host connected, authenticated, and then did not answer the + /// request to open a tunnel to the next hop. + /// + /// Its own variant rather than `HandshakeTimeout`, which names the target — + /// and the target is not the party that went quiet here. Sending an + /// operator to the wrong machine's logs is the mistake `CertificateRefused` + /// was added to stop. + #[error("A jump host did not open the tunnel to the next hop in time")] + TunnelOpenTimeout { host: String }, + #[error(transparent)] Warpgate(#[from] WarpgateError), } +impl ConnectionError { + pub fn client_message(&self) -> String { + match self { + ConnectionError::Vault(e) => e.client_message().to_string(), + ConnectionError::Aws(e) => e.client_message().to_string(), + ConnectionError::Authentication(reason) => { + format!("SSH target rejected Warpgate's authentication request: {reason}") + } + ConnectionError::CertificateRefused(reason) => { + format!("Warpgate refused the certificate issued for this session: {reason}") + } + ConnectionError::HostKeyMismatch { .. } => "Host key mismatch".to_string(), + ConnectionError::Resolve => "Could not resolve target address".to_string(), + ConnectionError::Aborted => "Connection aborted".to_string(), + ConnectionError::HandshakeTimeout => { + "The SSH target accepted the connection but never completed the handshake" + .to_string() + } + ConnectionError::TargetAuthenticationTimeout => { + "The SSH target accepted Warpgate's authentication request and never answered it" + .to_string() + } + ConnectionError::Internal => "Internal connection error".to_string(), + ConnectionError::JumpHostTargetNotFound => "Jump host target not found".to_string(), + ConnectionError::UntrustedJumpHost => { + "A jump host on the way to this target has an untrusted host key; check that host first" + .to_string() + } + ConnectionError::AuthenticationTimeout => { + "Authentication did not complete in time — the target or, for a certificate target, the issuer" + .to_string() + } + ConnectionError::TunnelOpenTimeout { host } => { + // `{:?}`, as the other arms that carry a configured string do. + // The host comes from a target's configuration, so it reaches a + // terminal belonging to whoever connects rather than to whoever + // set it — the same class as the target names already fixed, and + // missed here because this variant was added by a later fix. + format!("The jump host {host:?} did not open the tunnel to the next hop in time") + } + // Split out from the protocol errors below, for the admin + // host-key-check endpoint. Everywhere else this function's caller is + // an unauthenticated party and one flat category is right; there the + // caller is an authenticated operator, and "I cannot reach this host + // at all" and "this host's key is not trusted" are the same sentence + // today while being two entirely different jobs. Nothing is + // disclosed by separating them — the operator supplied the address. + // + // The kind, not the operating system's own string: this is the + // sanitiser, and a fixed set of phrases cannot carry anything + // through it. + ConnectionError::Io(e) | ConnectionError::Ssh(russh::Error::IO(e)) => { + format!( + "Could not open an SSH connection to the target: {}", + unreachable_reason(e.kind()) + ) + } + ConnectionError::Key(_) | ConnectionError::Ssh(_) => { + "SSH protocol error".to_string() + } + // Not `to_string()`. This is the sanitiser, and that arm passed + // `WarpgateError`'s `Display` through verbatim to a PTY and a + // browser — reachable from inside `authenticate_session`, where a + // database failure renders as `database error: {DbErr}` carrying SQL + // text, and an encryption-key mismatch names the configured key + // fingerprints. The one arm of the function that did not do the job + // the function exists for. + ConnectionError::Warpgate(_) => "Internal connection error".to_string(), + } + } +} + +/// Push the handshake bound out while a person is being asked about a host key. +/// +/// A named operation rather than a `reset` written out at the call site, so the +/// pair below can be driven by a test. The test that guarded this compared +/// `once_the_host_key_is_answered()` against `HANDSHAKE_TIMEOUT` and never +/// called either function — it would have passed with a call site deleted, or +/// with the two durations swapped between them. +fn pause_for_host_key_question(deadline: Pin<&mut tokio::time::Sleep>) { + deadline.reset(tokio::time::Instant::now() + while_a_host_key_answer_is_outstanding()); +} + +/// And bring it back once the answer arrives: the rest of the handshake is the +/// target's again, so it gets the target's bound. +fn resume_after_host_key_answer(deadline: Pin<&mut tokio::time::Sleep>) { + deadline.reset(tokio::time::Instant::now() + once_the_host_key_is_answered()); +} + +/// One end of a certificate's validity window, for a message a person reads. +/// +/// `None` is the sentinel meaning "no bound", which a certificate this feature +/// issues never carries. +/// +/// The formatting is fallible and that is handled, which is not fussiness. +/// `humantime`'s `Display` returns `Err` rather than truncating for any time at +/// or after the year 9999, and `to_string()` **panics** when a `Display` +/// errors. A certificate marked never-expiring lands exactly there — so +/// building this diagnostic killed the session's task before +/// `certificate_mismatch` could run the check that refuses such a certificate, +/// and the client was left holding a connection nobody would ever answer. +/// Found by unskipping the integration test that had been parked on "holds the +/// session open for a reason not yet isolated". This was the reason. +fn describe_certificate_time(at: Option) -> String { + use std::fmt::Write as _; + + let Some(at) = at else { + return "unbounded".to_owned(); + }; + let mut rendered = String::new(); + if write!(rendered, "{}", humantime::format_rfc3339_seconds(at)).is_err() { + return "a date beyond any this can render".to_owned(); + } + rendered +} + +/// Why a connection could not be opened, in a fixed set of words. +/// +/// Deliberately not `io::Error`'s own `Display`. This feeds the sanitiser, and +/// the whole point of that function is that nothing reaches a caller except +/// text this file chose. +const fn unreachable_reason(kind: std::io::ErrorKind) -> &'static str { + match kind { + std::io::ErrorKind::ConnectionRefused => "it refused the connection", + std::io::ErrorKind::TimedOut => "it did not answer in time", + std::io::ErrorKind::HostUnreachable | std::io::ErrorKind::NetworkUnreachable => { + "there is no route to it" + } + std::io::ErrorKind::ConnectionReset | std::io::ErrorKind::ConnectionAborted => { + "it closed the connection" + } + _ => "the connection could not be established", + } +} + +/// Vault is a second gate, not a trusted party, so what comes back is checked +/// against what was asked for before it is offered to the target. +/// +/// The type and key checks are belt-and-braces — neither could authenticate +/// anything — but the last two are load-bearing. The target's sshd enforces +/// whatever critical options arrive, so a `force-command` planted on a role +/// replaces what the user asked to run while keeping their principal and key ID +/// on it: the target's own log then attributes the attacker's command to them. +/// Writing a role is a lower bar than signing with it, so this is the only place +/// the check can land. +/// How much longer than the requested TTL a certificate may be valid for +/// before it is refused. +/// +/// The window is measured from now, and the certificate was signed a moment +/// earlier, so an honest issuer always comes back at or under what was asked +/// for. The slack is for the clocks disagreeing, not for the issuer being +/// generous. +const CERTIFICATE_TTL_SLACK: Duration = Duration::from_secs(60); + +/// Stands in the username field when the session has no user info recorded. +/// +/// A user of this name is moved aside by `user_key_id_field`, for the same +/// reason an attribution is: otherwise a session driven by a real person reads +/// in the target's log exactly like one with no user recorded at all. +/// +/// The comment here used to say `key_id_field` rejected a colon, so nothing +/// could collide. It rejected nothing — it substituted — and the collision that +/// threatens this constant is a username equal to it, which colons have no part +/// in. Two wrong statements holding each other up. +const UNATTRIBUTED: &str = "unattributed"; + +/// A username as it may appear in a key ID field. +/// +/// The key ID is `warpgate::`, three fields split on a +/// colon, and the target's own sshd log carries it verbatim — a name with a +/// colon in it shifts every field, so whatever reads that log names the wrong +/// person. The admin API refuses one now; a name arriving from an IdP never +/// passes through the admin API, so the structure is held here too. +/// +/// A name equal to an attribution is held here for the same reason and by the +/// same argument. `attribution()` puts `admin-token` in this field when the +/// admin API token drives a session, so a user of that name reads in the +/// target's log exactly as the token does. The admin API refuses it — and is +/// one of six paths that create a user. SSO auto-provisioning inserts the +/// IdP's `preferred_username` directly, and the two CLI commands insert what +/// the operator typed, so the refusal is not on the path that matters. +/// +/// Substituted rather than refused: a name from a directory is not something +/// the connecting user can fix mid-session. +/// +/// Percent-encoded rather than replaced. `.replace(':', "_")` mapped +/// `root:admin` and `root_admin` onto the same field, so the log line this +/// whole feature exists to produce could name a person who did not connect — +/// the one claim the feature makes, lost to a one-character substitution. `%` +/// is encoded first, or a literal `%3A` in a name would read back as a colon. +fn key_id_field(name: &str) -> String { + name.replace('%', "%25").replace(':', "%3A") +} + +/// A name that came from a person, as it may appear in a key ID field. +/// +/// Everything `key_id_field` does, and one thing more: a name equal to an +/// attribution is moved aside, because `attribution()` puts those in this very +/// field when a token drives a session. A user of that name would otherwise be +/// indistinguishable from the gateway in the target's sshd log and in Vault's. +/// +/// Only for names that came from a person. The gateway's own attribution goes +/// through `key_id_field` untouched — substituting it renames the thing it +/// identifies, which is exactly what the first version of this did: two guards +/// failed their baseline because `admin-token` had become `admin-token_`. +fn user_key_id_field(name: &str) -> String { + let field = key_id_field(name); + if is_reserved_key_id_field(&field) { + return format!("{field}_"); + } + field +} + +/// A field the gateway itself puts in a key ID, which a person must not occupy. +/// +/// One list, because there were two and only one of them was consulted: +/// `attribution()`'s token names were held, and `UNATTRIBUTED` — written into +/// the same field by the same code — was not. +fn is_reserved_key_id_field(field: &str) -> bool { + TOKEN_ATTRIBUTIONS.contains(&field) || field == UNATTRIBUTED +} + +/// Whether a host-key check can be answered by this chain at all. +/// +/// The walk decides each hop's role by identity, so a `check_target` naming no +/// hop leaves every one of them `TraversedWhileChecking`: none reports its key, +/// none stops the walk, and the caller is handed a live session and no answer. +/// Checked before the first connection rather than after the last, because a +/// question this chain cannot answer is not worth opening a socket for. +/// +/// Unreachable while the chain is built from the target being asked about, +/// which is exactly the assumption deciding roles by identity exists to stop +/// depending on. +fn chain_can_answer(check_target: Option, hops: &[Uuid]) -> bool { + check_target.is_none_or(|asked_about| hops.contains(&asked_about)) +} + +/// Whether the certificate was signed by the CA the operator pinned. +/// +/// Separate from `certificate_mismatch` because it asks a different question. +/// Every check there compares the response against the request — is this the +/// key ID we asked for, the principal, the window. This one asks who signed it, +/// which the request cannot answer. +/// +/// A pinned key that will not parse is a refusal, not a warning skipped over: a +/// typo in the config would otherwise turn the check off silently, which is the +/// failure mode a pin exists to prevent. +fn certificate_signer_mismatch(certificate: &Certificate, pinned: Option<&str>) -> Option { + let pinned = pinned?; + let Ok(expected) = PublicKey::from_openssh(pinned) else { + return Some( + "The pinned Vault CA public key in the configuration could not be parsed".to_owned(), + ); + }; + if certificate.signature_key() == expected.key_data() { + return None; + } + Some("Vault issued a certificate signed by a CA other than the pinned one".to_owned()) +} + +fn certificate_mismatch( + certificate: &Certificate, + key: &PublicKey, + principal: &str, + key_id: &str, + allowed_options: &[SshCertificateCriticalOption], + allowed_extensions: &[String], + requested_ttl: Option, +) -> Option { + // The target's sshd logs this verbatim, and a session being attributable to + // a person from the target side alone is the whole claim this feature + // makes. An issuer that returns a different one — or a very long one — + // breaks the attribution and floods the target's log with it. + if certificate.key_id() != key_id { + return Some( + "Vault issued a certificate under a key ID other than the one requested".to_owned(), + ); + } + if certificate.cert_type() != CertType::User { + return Some("Vault returned a host certificate rather than a user certificate".to_owned()); + } + if certificate.public_key() != key.key_data() { + return Some("Vault signed a key other than the one generated for this session".to_owned()); + } + // Vault returns the requested set verbatim — trimmed, deduped and sorted — + // or refuses outright, so anything other than exactly the account that was + // asked for means the answer did not come from this request. + // + // Exactly, rather than merely containing it. An extra principal is another + // account the target will accept this certificate for, chosen by whoever + // answered rather than by the operator, and where sshd maps principals + // through `AuthorizedPrincipalsFile` it need not resemble a username at all. + // It is also the shape of CVE-2026-35414, where a comma inside a principal + // splits one name into two for one of sshd's checks and not the other. + let principals = certificate.valid_principals(); + if principals.len() != 1 || principals.first().is_none_or(|only| only != principal) { + return Some(format!( + "Vault issued a certificate naming {:?} rather than only the target account {principal}", + principals + )); + } + + // The window is the feature's whole premise: a credential too short-lived + // to be worth stealing. Nothing checked it, so a role with a `max_ttl` of + // years — or one quietly edited to have one — produced a certificate good + // for years, and every layer downstream accepted it. + // Every way this can be wrong, because the first version handled only the + // ordinary one and let the other three past. + // + // `u64::MAX` is OpenSSH's "never expires" sentinel (PROTOCOL.certkeys), and + // it is what `ssh-keygen -V always:forever` and a Vault role with no TTL + // both write. It is checked on the raw field rather than through + // `valid_before_time()`, which reports the sentinel as a real instant + // capped at `i64::MAX` — refused either way, but as "valid for + // 2562047787518949 hours", which names the wrong problem. + // + // `valid_before_time()` returns `None` only for a value above `i64::MAX` + // that is *not* the sentinel. No tool produces one; an issuer that wants + // this certificate not to expire can write one by hand. + if certificate.valid_before() == u64::MAX { + return Some( + "Vault issued a certificate that never expires, which is not a session credential" + .to_owned(), + ); + } + match certificate + .valid_before_time() + .map(|at| at.duration_since(std::time::SystemTime::now())) + { + None => { + return Some( + "Vault issued a certificate with an unrepresentable expiry time".to_owned(), + ); + } + Some(Err(_)) => { + return Some( + "Vault issued a certificate that has already expired; check the clock on this host" + .to_owned(), + ); + } + Some(Ok(lifetime)) if lifetime > MAX_CERTIFICATE_LIFETIME => { + return Some(format!( + "Vault issued a certificate valid for {} hours, far longer than a session credential should be", + lifetime.as_secs() / 3600 + )); + } + // The ceiling is a backstop against a misconfigured role. This is the + // operator's own number, and it was sent and then never looked at + // again: a target configured for ninety seconds accepted a certificate + // good for twenty-three hours, because that is still under the ceiling. + // Asking is not getting, and every other field Vault returns is checked + // against what was asked for. + Some(Ok(lifetime)) + if requested_ttl.is_some_and(|ttl| lifetime > ttl + CERTIFICATE_TTL_SLACK) => + { + return Some(format!( + "Vault issued a certificate valid for {}s, longer than the {}s this target asked for", + lifetime.as_secs(), + requested_ttl.map_or(0, |ttl| ttl.as_secs()) + )); + } + Some(Ok(_)) => {} + } + + // Both directions, and pinning a value is what decides which. + // + // The list is an allow-list: an option that is not on it is refused. That + // was built against someone with write access to a Vault role but no right + // to sign with it, adding an option nobody asked for. Approached from the + // other side they *remove* one instead — a target whose whole point is a + // pinned `force-command` accepting a certificate carrying none at all is a + // full shell rather than the one command, and checking only what arrived + // can never see that. + // + // So a pinned entry is also mandatory. Pinning a value is the act of saying + // what the option must be, which is not something a certificate can satisfy + // by leaving it out. A bare name only permits: it is how a role that + // *sometimes* sets an option is expressed, which an all-mandatory list + // could not express at all — list the option and certificates without it + // fail, omit it and certificates with it fail. + for expected in allowed_options { + if expected.value.is_some() && !certificate.critical_options().contains_key(&expected.name) + { + return Some(format!( + "Vault issued a certificate without the critical option {:?}, which this target pins to a specific value", + expected.name + )); + } + } + + // Extensions, which until now were logged and nothing else. + // + // The argument for checking critical options — writing a Vault role is a + // lower bar than signing with it, so this is the only place it can be + // caught — applies to extensions unchanged. They are a separate + // authorization mechanism, and the one that decides whether a session can + // forward ports or reach the connecting user's agent. A `force-command` + // covers the shell and exec channels; `direct-tcpip` and agent forwarding + // are judged by OpenSSH purely on what the certificate carries. + // + // No second direction here, unlike critical options: an extension that is + // absent grants nothing, so there is nothing to remove that would widen + // access. + for name in certificate.extensions().keys() { + if !allowed_extensions.iter().any(|allowed| allowed == name) { + return Some(format!( + "Vault issued a certificate carrying the extension {name:?}, which this target does not allow" + )); + } + } + + for (name, value) in certificate.critical_options().iter() { + // The *strictest* matching entry, not the first one. + // + // `.find()` returned whichever the operator happened to type first. A + // config naming `force-command` twice — once bare, once pinned to a + // value — passed the mandatory-presence loop above on the pinned entry + // and then matched the bare one here, so any command at all was + // accepted while the admin UI showed a pin. Fails open, silently, and + // an operator with two rows on screen has no way to see which one is + // deciding. + // + // Duplicates are a configuration mistake rather than a feature, and + // every pin is enforced rather than one of them being picked. That also + // settles two pins that disagree: no value satisfies both, so the + // certificate is refused instead of one entry winning by position. + let mut named = allowed_options + .iter() + .filter(|option| &option.name == name) + .peekable(); + if named.peek().is_none() { + // Quoted with `{:?}`, which escapes control characters: this string + // comes out of the certificate and ends up on the connecting user's + // terminal, where a raw escape sequence would be executed rather + // than shown. + return Some(format!( + "Vault issued a certificate carrying the critical option {name:?}, which this target does not allow" + )); + } + for option in named { + if let Some(expected) = &option.value + && expected != value + { + return Some(format!( + "Vault issued a certificate whose critical option {name:?} does not match the value configured for this target" + )); + } + } + } + + None +} + +#[derive(Clone, Debug)] pub struct ResolvedSshChainHost { + /// Which target this hop is. + /// + /// Carried past resolution because the connection code needs to know which + /// hop was asked about, and used to infer it from position: the last one. + /// That is true of every chain built today and is an assumption rather than + /// a fact — the identity exists here and was thrown away one line later. + pub id: Uuid, pub name: String, pub ssh_options: TargetSSHOptions, } @@ -158,6 +818,7 @@ pub async fn resolve_ssh_chain( } jumps.push(ResolvedSshChainHost { + id: t.id, name: t.name.clone(), ssh_options: opts, }); @@ -235,9 +896,40 @@ impl RCEvent { pub type RCCommandReply = oneshot::Sender>; +/// Who asked, when no session user can be named. +/// +/// Two kinds of name arrive here and they are indistinguishable as strings, +/// which is the whole of the defect this exists to close. Carried as data +/// instead of guessed from the text. +#[derive(Clone, Debug)] +pub enum IdentityHint { + /// The gateway's own attribution — `admin-token`, `cluster-token`. Kept + /// verbatim: it is the string a reader trusts to mean that no person did + /// this. + Gateway(String), + /// A person, named by whatever authenticated them. Sanitised like any + /// other name a person chose. + Person(String), +} + #[derive(Clone, Debug)] pub enum RCCommand { - Connect(Vec), + Connect(Vec), + /// Connect only as far as the target's host key, then stop. Carries who + /// asked, because any jump host on the way is still authenticated and the + /// certificate that authenticates it has to name a person. + /// + /// A separate command rather than the caller dropping its handle once it has + /// what it wants: the connection otherwise runs on into authentication, and + /// for a certificate target that means a real certificate issued and a real + /// session opened for nobody. Cancelling on a dropped handle is a race the + /// caller loses about half the time; refusing to start is not. + CheckHostKey { + chain: Vec, + /// The hop the caller asked about, by identity rather than by position. + target_id: Uuid, + requested_by: IdentityHint, + }, Channel(Uuid, ChannelOperation), ForwardTCPIP(String, u32), CancelTCPIPForward(String, u32), @@ -274,6 +966,16 @@ pub struct RemoteClient { inner_event_tx: UnboundedSender, child_tasks: Vec>>, services: Services, + /// Who to name in a certificate when there is no session to look up. + /// + /// The admin host-key check has no session — it is a button press, not a + /// login — but reaching a target behind a jump host still authenticates + /// that hop, and a certificate is minted for it. Without this the key ID + /// falls back to naming the random UUID that stood in for a session, so the + /// jump host's sshd log and Vault's issuance log both record a certificate + /// that resolves to nobody. That is the attribution failure the whole + /// feature exists to prevent, in the one caller that has no user to look up. + identity_hint: Option, } pub struct RemoteClientHandles { @@ -303,6 +1005,7 @@ impl RemoteClient { inner_event_tx: inner_event_tx.clone(), child_tasks: vec![], services, + identity_hint: None, abort_rx, }; @@ -411,7 +1114,15 @@ impl RemoteClient { break } } - Some(()) = self.abort_rx.recv() => { + // `_` rather than `Some(())`, matching the connect + // loop. The two are not two spellings of one thing: + // under `Some(())` a closed `abort_rx` disables this + // branch instead of firing it, and with the event + // branch disabled too there is nothing left for + // `select!` to wait on. Every sender being gone is + // an owner that dropped without disconnecting, which + // is what this branch is for. + _ = self.abort_rx.recv() => { debug!("Abort requested"); self.disconnect().await; break @@ -528,13 +1239,34 @@ impl RemoteClient { } } Err(e) => { - debug!("Connect error: {}", e); + // `{:?}` rather than `{}` throughout this file for + // anything carrying a remote party's words. A newline in a + // Vault error body or an unresolved host name forges a + // whole record in the default text format — a log line the + // reader has no way to tell from one Warpgate wrote. Debug + // escapes it; Display does not, and `emit_pty_output`'s + // escaping does not reach here because no `tracing` call + // routes through it. + debug!("Connect error: {e:?}"); let _ = self.tx.send(RCEvent::ConnectionError(e)).await; self.set_disconnected().await; return Ok(true); } }, + RCCommand::CheckHostKey { + chain, + target_id, + requested_by, + } => { + self.identity_hint = Some(requested_by); + if let Err(e) = self.check_host_key(chain, target_id).await { + debug!("Host key check error: {e:?}"); + let _ = self.tx.send(RCEvent::ConnectionError(e)).await; + } + self.set_disconnected().await; + return Ok(true); + } RCCommand::Channel(ch, op) => { self.apply_channel_op(ch, op).await?; } @@ -644,13 +1376,37 @@ impl RemoteClient { /// Connect through a pre-resolved chain of SSH hops, each tunnelled through the previous. /// `chain` must be non-empty; the first entry is connected directly, subsequent ones via /// `channel_open_direct_tcpip` through the previous session. + /// `checking_host_key` names the last hop as the one being asked about: the + /// intermediate ones must still authenticate, or there is no tunnel to carry + /// the last one, and their keys are not the answer to the question. + /// Returns `None` when it stopped at that hop's host key as asked. async fn connect_chain( &mut self, - chain: Vec, - ) -> Result<(Handle, UnboundedReceiver), ConnectionError> - { + chain: Vec, + // `check_target`: the hop being asked about, when this is a host-key + // check, by identity. Deciding it by position asserts that the chain + // always ends at the target that was named — true of every chain built + // today, and an assumption rather than something checked. + check_target: Option, + ) -> Result< + Option<(Handle, UnboundedReceiver)>, + ConnectionError, + > { + if !chain_can_answer( + check_target, + &chain.iter().map(|hop| hop.id).collect::>(), + ) { + error!( + ?check_target, + "The chain does not contain the host that was asked about" + ); + return Err(ConnectionError::Resolve); + } + let mut iter = chain.into_iter(); - let first = iter.next().ok_or(ConnectionError::Resolve)?; + let first_hop = iter.next().ok_or(ConnectionError::Resolve)?; + let first_id = first_hop.id; + let first = first_hop.ssh_options; let config = self.build_ssh_config(&first).await; let address_str = format!("{}:{}", first.host, first.port); @@ -658,8 +1414,8 @@ impl RemoteClient { .to_socket_addrs() .map_err(ConnectionError::Io) .and_then(|mut x| x.next().ok_or(ConnectionError::Resolve)) - .inspect_err(|e| error!(?e, address=%address_str, "Cannot resolve address"))?; - info!(?address, username = %first.username, "Connecting"); + .inspect_err(|e| error!(?e, address = ?address_str, "Cannot resolve address"))?; + info!(?address, username = ?first.username, "Connecting"); let (event_tx, event_rx) = unbounded_channel(); let handler = ClientHandler { ssh_options: first.clone(), @@ -668,27 +1424,50 @@ impl RemoteClient { session_id: self.id, }; let fut = russh::client::connect(config, address, handler); - let (mut session, mut active_rx) = self - .wait_for_connection(&first, fut, event_rx, false) + let Some((mut session, mut active_rx)) = self + .wait_for_connection(&first, fut, event_rx, role(check_target, first_id)) .boxed() - .await?; + .await? + else { + return Ok(None); + }; - for ssh_options in iter { + for hop in iter { + let hop_id = hop.id; + let ssh_options = hop.ssh_options; let _ = self.tx.send(RCEvent::HopConnected).await; info!( - host = %ssh_options.host, + host = ?ssh_options.host, port = ssh_options.port, "Opening direct-tcpip channel through jump host" ); - let channel = session - .channel_open_direct_tcpip( + // Bounded, because nothing else bounds it. + // + // Each hop's own handshake deadline is armed inside + // `wait_for_connection`, which runs *after* this — so the step that + // asks a jump host to open the tunnel had no limit at all. A jump + // host that accepts the request and never replies stalls here for as + // long as the previous hop's inactivity timeout allows, which is + // five minutes by default and hours wherever an operator has raised + // it. Reachable from the admin host-key check too, which adds no + // timeout of its own. + let channel = tokio::time::timeout( + HANDSHAKE_TIMEOUT, + session.channel_open_direct_tcpip( ssh_options.host.clone(), u32::from(ssh_options.port), "localhost".to_string(), 0, - ) - .await - .map_err(ConnectionError::Ssh)?; + ), + ) + .await + .map_err(|_| { + error!(host = ?ssh_options.host, "Jump host did not open the tunnel in time"); + ConnectionError::TunnelOpenTimeout { + host: ssh_options.host.clone(), + } + })? + .map_err(ConnectionError::Ssh)?; let stream = channel.into_stream(); let config = self.build_ssh_config(&ssh_options).await; let (event_tx, event_rx) = unbounded_channel(); @@ -699,19 +1478,43 @@ impl RemoteClient { session_id: self.id, }; let fut = russh::client::connect_stream(config, stream, handler); - let (new_session, new_rx) = self - .wait_for_connection(&ssh_options, fut, event_rx, false) + let Some((new_session, new_rx)) = self + .wait_for_connection(&ssh_options, fut, event_rx, role(check_target, hop_id)) .boxed() - .await?; + .await? + else { + return Ok(None); + }; session = new_session; active_rx = new_rx; } - Ok((session, active_rx)) + Ok(Some((session, active_rx))) } - async fn connect(&mut self, chain: Vec) -> Result<(), ConnectionError> { - let (session, mut event_rx) = self.connect_chain(chain).boxed().await?; + /// Connects as far as the target's host key and stops, for the admin-side + /// check. Nothing is authenticated, so nothing is issued. + async fn check_host_key( + &mut self, + chain: Vec, + target_id: Uuid, + ) -> Result<(), ConnectionError> { + if let Some((session, _)) = self.connect_chain(chain, Some(target_id)).boxed().await? { + // Only reachable if the connection came up without the handler ever + // reporting a host key, which should not happen — but an open + // session left behind would be exactly the leak this command exists + // to close. + let _ = session + .disconnect(russh::Disconnect::ByApplication, "", "") + .await; + } + Ok(()) + } + + async fn connect(&mut self, chain: Vec) -> Result<(), ConnectionError> { + let Some((session, mut event_rx)) = self.connect_chain(chain, None).boxed().await? else { + return Err(ConnectionError::Internal); + }; self.session = Some(Arc::new(Mutex::new(session))); @@ -739,29 +1542,160 @@ impl RemoteClient { ssh_options: &TargetSSHOptions, fut_connect: Fut, mut event_rx: UnboundedReceiver, - _is_jump_host: bool, - ) -> Result<(Handle, UnboundedReceiver), ConnectionError> + hop: HopRole, + ) -> Result< + Option<(Handle, UnboundedReceiver)>, + ConnectionError, + > where Fut: Future, ClientHandlerError>>, { pin_mut!(fut_connect); + // Nothing else bounds this. russh limits how long an identification + // string may be, but not how long a peer may take to send the rest of + // the handshake, and the inactivity timeout only starts once the + // session loop is running. A target that completes the TCP connection + // and then goes quiet therefore held the session — and its task, and + // its slot — for as long as it liked. Reachable by anyone who can start + // a session to a target that has been compromised, or merely wedged. + // + // Its own bound, rather than the inactivity timeout. Measured: with + // that timeout at 45s a stalled handshake was held for 55s, so it was + // being bounded by the *inbound* session's patience — which governs how + // long an idle interactive session may live and is legitimately raised + // to hours. Borrowing it here would extend this hold to match. + let handshake_deadline = tokio::time::sleep(HANDSHAKE_TIMEOUT); + pin_mut!(handshake_deadline); + + // The deadline pauses while a host key is outstanding and resumes the + // moment it is answered, which requires seeing the answer. The reply + // channel goes to the session, so it is intercepted: the session is + // handed a substitute sender, and the real one is held here until the + // answer arrives on `host_key_answers`. + // + // The first version disarmed the deadline unconditionally and never + // re-armed it, on the reasoning that "from here the wait is a person's". + // That reasoning is not available at this point in the code: the + // verification mode is read in `handle_unknown_host_key`, which answers + // instantly under `AutoAccept` and `AutoReject`. A target could present + // an unknown host key under auto-accept, cancel the deadline for good, + // and then go silent — the exact hold this bound exists to catch, + // reintroduced by the fix for a different problem. + let (host_key_answer_tx, mut host_key_answers) = tokio::sync::mpsc::channel::(1); + let mut outstanding_host_key: Option> = None; + loop { tokio::select! { Some(event) = event_rx.recv() => { match event { ClientHandlerEvent::HostKeyReceived(key) => { - self.tx.send(RCEvent::HostKeyReceived(key, ssh_options.host.clone(), ssh_options.port)).await.map_err(|_| ConnectionError::Internal)?; + // Every hop presents a key, and the caller asked + // about one of them. Reporting an intermediate hop's + // key answers a question nobody asked. + // + // Both sides of #2437 are kept. The hop's address + // now travels with the key, so the admin endpoint + // can say which host it is looking at; and which hop + // answers is still decided here, by identity. Two + // hops can share an address, and only the identity + // says which one the caller named. + if hop.reports_host_key() { + self.tx.send(RCEvent::HostKeyReceived(key, ssh_options.host.clone(), ssh_options.port)).await.map_err(|_| ConnectionError::Internal)?; + } + if hop.stops_after_host_key() { + return Ok(None); + } } ClientHandlerEvent::HostKeyUnknown(key, reply) => { - self.tx.send(RCEvent::HostKeyUnknown(key, ssh_options.host.clone(), ssh_options.port, reply)).await.map_err(|_| ConnectionError::Internal)?; + if hop.reports_host_key() { + // Paused, not disarmed, and only for as long as + // the answer is outstanding. + // + // `Prompt` is the default verification mode, so + // on a stock install the first connection to any + // target — and any target that has rotated its + // key — reaches this line. Leaving the deadline + // armed gave whoever is comparing a base64 + // fingerprint under thirty seconds to do it, and + // then failed the connection with "the target + // never completed the handshake", naming the + // target for a delay that was the user's. + // + // It also could not converge: `known_hosts.trust()` + // runs in `check_server_key` *after* the reply, + // so a fired deadline drops the connection future, + // the reply channel dies, and the key is never + // stored — the next attempt asks again. + // + // Under the automatic modes the answer comes back + // in microseconds and the pause is not observable, + // which is the point: this arm no longer needs to + // know which mode is configured to stay bounded. + pause_for_host_key_question(handshake_deadline.as_mut()); + let (intercept_tx, intercept_rx) = oneshot::channel(); + outstanding_host_key = Some(reply); + let answer_tx = host_key_answer_tx.clone(); + tokio::spawn(async move { + if let Ok(answer) = intercept_rx.await { + let _ = answer_tx.send(answer).await; + } + }); + self.tx.send(RCEvent::HostKeyUnknown(key, ssh_options.host.clone(), ssh_options.port, intercept_tx)).await.map_err(|_| ConnectionError::Internal)?; + } else { + // Nobody is listening for an answer on this hop, + // and the handler is waiting for one. Refusing is + // the honest reply: a jump host whose key is not + // yet trusted has its own check to pass first, + // and silently accepting it here would trust it + // on the strength of a question about something + // else. + let _ = reply.send(false); + return Err(ConnectionError::UntrustedJumpHost); + } + if hop.stops_after_host_key() { + return Ok(None); + } } _ => {} } } - Some(()) = self.abort_rx.recv() => { + // The answer to a host key question, on its way back to + // `check_server_key`. The remaining handshake is the target's + // again from here, so the bound comes back with it. + Some(answer) = host_key_answers.recv() => { + resume_after_host_key_answer(handshake_deadline.as_mut()); + if let Some(reply) = outstanding_host_key.take() { + // A closed receiver means the connection future is + // already gone; there is nothing left to answer. + let _ = reply.send(answer); + } + } + () = &mut handshake_deadline => { + error!(host = ?ssh_options.host, "Target did not finish the SSH handshake in time"); + // No `set_disconnected()` here: `handle_command` calls it + // immediately after sending the error, so doing it first + // only reorders `Done` ahead of the reason. A review argued + // that ordering discards the message, since both session + // loops treat `Done` as terminal — I could not reproduce + // that, the reason arrives either way, so this is tidiness + // rather than a fix. The test below pins the message + // regardless of which explanation is right. + return Err(ConnectionError::HandshakeTimeout); + } + // `None` means every sender is gone, so the owner has dropped + // without disconnecting. `ServerSession::drop` signals first, so + // a live session never arrives here still wanting the + // connection — anything else that does is abandoning it. + _ = self.abort_rx.recv() => { info!("Abort requested"); - self.set_disconnected().await; + // No `set_disconnected()` here, for the reason the + // `HandshakeTimeout` branch above gives: it sends `Done`, + // and `handle_command` calls it anyway immediately after + // sending the error. Doing it here only puts `Done` ahead + // of the reason on the same channel. This branch was + // reported as fixed while still holding that ordering — the + // pattern was corrected and the ordering was not. return Err(ConnectionError::Aborted) } session = &mut fut_connect => { @@ -778,20 +1712,102 @@ impl RemoteClient { } }; - self.authenticate_session( - &mut session, - &ssh_options.host, - &ssh_options.username, + // Under the same deadline. + // + // `tokio::select!` stops polling its other branches once it + // commits to one, so awaiting authentication inside this arm + // put it outside the deadline entirely — and authentication + // is where the Vault round trip happens and where a + // certificate is minted. A target that finished the + // transport handshake and then never answered + // `SSH_MSG_USERAUTH_REQUEST` held the task, the session slot, + // the ephemeral key and a live certificate until russh's + // inactivity timeout — five minutes by default and hours + // wherever an operator has raised it for interactive use. + // That is the exact hold this deadline was added to bound, + // one stage further along than the stage it was bounding. + // + // Abort is not polled across this window: `abort_rx` needs + // `&mut self` and `authenticate_session` takes `&self`. It + // was not polled here before either, and the window it + // covers is now bounded, so the cost is up to + // `HANDSHAKE_TIMEOUT` of delay in tearing down a connection + // whose owner has already gone. + // Its own budget, not the transport handshake's. + let budget = authentication_budget( &ssh_options.auth, - ssh_options.allow_insecure_algos.unwrap_or(false) - ).await?; + self.services.vault.get().map(|v| v.timeout()), + ); + let authentication_deadline = tokio::time::sleep(budget); + pin_mut!(authentication_deadline); + + tokio::select! { + () = &mut authentication_deadline => { + error!( + host = ?ssh_options.host, + budget = ?budget, + "Authentication did not finish in time" + ); + return Err(ConnectionError::AuthenticationTimeout); + } + result = self.authenticate_session( + &mut session, + &ssh_options.host, + &ssh_options.username, + &ssh_options.auth, + ssh_options.allow_insecure_algos.unwrap_or(false) + ) => result?, + } - return Ok((session, event_rx)); + return Ok(Some((session, event_rx))); } } } } + /// Identifies the Warpgate session in the certificate's key ID. The target's + /// own sshd logs it verbatim, which is what lets a proxied session be traced + /// to a person from the target side alone. + async fn certificate_key_id(&self) -> String { + let session = self + .services + .state + .lock() + .await + .sessions + .get(&self.id) + .cloned(); + + let username = match session { + Some(session) => session + .lock() + .await + .user_info + .as_ref() + .map(|user| user.username.clone()), + None => None, + }; + + let username = match username { + Some(name) => Some(user_key_id_field(&name)), + None => self.identity_hint.as_ref().map(|hint| match hint { + IdentityHint::Gateway(name) => key_id_field(name), + IdentityHint::Person(name) => user_key_id_field(name), + }), + }; + // Three fields either way. Dropping the middle one shifted the session + // UUID into the position a reader takes for the username — so a log + // line naming nobody was indistinguishable from one naming a user + // called `0e5f…`, and the field this whole feature exists to fill was + // silently wrong rather than visibly absent. Reachable whenever the + // session's user info has not been recorded, e.g. a transient database + // failure in `set_user_info`. + username.map_or_else( + || format!("warpgate:{}:{}", UNATTRIBUTED, self.id), + |username| format!("warpgate:{username}:{}", self.id), + ) + } + async fn authenticate_session( &self, session: &mut Handle, @@ -805,9 +1821,10 @@ impl RemoteClient { match auth { SSHTargetAuth::Password(auth) => { let password = auth.password.reveal().map_err(WarpgateError::from)?; - let response = session - .authenticate_password(username.to_string(), password.expose_secret()) - .await?; + let response = bounded_userauth( + session.authenticate_password(username.to_string(), password.expose_secret()), + ) + .await?; auth_result = self ._handle_auth_result(session, username.to_string(), response) .await @@ -834,12 +1851,11 @@ impl RemoteClient { continue; } let key_str = key.public_key().to_openssh().map_err(russh::Error::from)?; - let mut response = session - .authenticate_publickey( - username.to_string(), - PrivateKeyWithHashAlg::new(key.clone(), best_hash), - ) - .await?; + let mut response = bounded_userauth(session.authenticate_publickey( + username.to_string(), + PrivateKeyWithHashAlg::new(key.clone(), best_hash), + )) + .await?; auth_result = self ._handle_auth_result(session, username.to_string(), response) @@ -851,12 +1867,11 @@ impl RemoteClient { && best_hash.is_some() && allow_insecure_algos { - response = session - .authenticate_publickey( - username.to_string(), - PrivateKeyWithHashAlg::new(key.clone(), None), - ) - .await?; + response = bounded_userauth(session.authenticate_publickey( + username.to_string(), + PrivateKeyWithHashAlg::new(key.clone(), None), + )) + .await?; auth_result = self ._handle_auth_result(session, username.to_string(), response) @@ -865,13 +1880,127 @@ impl RemoteClient { } if auth_result { - debug!(username=username, key=%key_str, "Authenticated with key"); + debug!(username = ?username, key = %key_str, "Authenticated with key"); break; } auth_error_msg = Some("Public key authentication was rejected by the SSH target".into()); } } + SSHTargetAuth::Certificate(auth) => { + if let Some(vault) = self.services.vault.get() { + // The key exists only for this authentication attempt — nothing + // the target will ever trust again outlives this scope. + let key = Arc::new( + PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .map_err(russh::keys::Error::from)?, + ); + let public_key = key + .public_key() + .to_openssh() + .map_err(russh::keys::Error::from)?; + let key_id = self.certificate_key_id().await; + let role = auth.role.as_deref().unwrap_or_else(|| vault.default_role()); + + let signed_key = vault + .sign_ssh_key(role, &public_key, username, &key_id) + .await?; + let certificate = + Certificate::from_openssh(&signed_key).map_err(russh::keys::Error::from)?; + + // Extensions do not change what runs, but an unexpected one + // still says the role is not what the operator thinks. + if !certificate.extensions().is_empty() { + debug!( + extensions = ?certificate.extensions().keys().collect::>(), + key_id, + "Certificate carries extensions" + ); + } + + // Captured before the certificate is handed to russh, which + // takes ownership of it. + let validity = ( + describe_certificate_time(certificate.valid_after_time()), + describe_certificate_time(certificate.valid_before_time()), + ); + + if let Some(reason) = + certificate_signer_mismatch(&certificate, vault.pinned_ca_public_key()) + .or_else(|| { + certificate_mismatch( + &certificate, + key.public_key(), + username, + &key_id, + &auth.allowed_critical_options, + &auth.allowed_extensions, + vault.certificate_ttl(), + ) + }) + { + // Surfaced to the user, not only to the log: a session + // that dies with "the target rejected you" sends whoever + // is debugging it to the wrong machine entirely. + warn!(key_id, reason, "Refusing the issued certificate"); + return Err(ConnectionError::CertificateRefused(reason)); + } else { + let response = bounded_userauth(session.authenticate_openssh_cert( + username.to_string(), + key, + certificate, + )) + .await?; + + // No `_handle_auth_result` here, deliberately. + // + // That helper falls through to keyboard-interactive and + // answers every prompt with an empty string. On a target + // whose `TrustedUserCAKeys` was never set, sshd refuses + // the certificate, offers keyboard-interactive, and a + // permissive PAM stack — `pam_permit`, `nullok`, the + // minimal stack in a lot of appliance images — accepts + // it. The session then proceeds, the log says + // "Authenticated with certificate", and the target's own + // sshd log carries no key ID at all: the attribution + // this feature exists to produce is silently absent, and + // the evidence says the opposite. + // + // A certificate target has one credential by design. If + // it is refused, that is the answer. + auth_result = matches!(response, AuthResult::Success); + + if auth_result { + debug!( + username = username, + key_id, "Authenticated with certificate" + ); + } else { + // The window is named because the most common cause + // of a target refusing an otherwise good certificate + // is its own clock: these live minutes, and a target + // that lags rejects them as not yet valid without + // saying so anywhere the operator can see. + auth_error_msg = Some(format!( + "Certificate authentication was rejected by the SSH target \ + (the certificate was valid from {} to {}; check the target's clock)", + validity.0, validity.1 + )); + } + } + } else { + // `CertificateRefused`, not `Authentication`: nothing was + // ever offered to the target, and the other variant renders + // as "SSH target rejected Warpgate's authentication + // request", which sends whoever is debugging this to the + // target's logs to look for a refusal that is not there. + // Reachable whenever a certificate target outlives the + // `vault:` section, or is reached from a node that has none. + return Err(ConnectionError::CertificateRefused( + "no Vault server is configured on this node".to_owned(), + )); + } + } SSHTargetAuth::IamRole(_) => { let instance_info = warpgate_aws::find_instance_by_ip(host).await?; @@ -898,12 +2027,11 @@ impl RemoteClient { // Now authenticate with this key (key is valid for 60 seconds) let key = Arc::new(key.clone()); let best_hash = session.best_supported_rsa_hash().await?.flatten(); - let response = session - .authenticate_publickey( - username.to_string(), - PrivateKeyWithHashAlg::new(key.clone(), best_hash), - ) - .await?; + let response = bounded_userauth(session.authenticate_publickey( + username.to_string(), + PrivateKeyWithHashAlg::new(key.clone(), best_hash), + )) + .await?; auth_result = self ._handle_auth_result(session, username.to_string(), response) @@ -928,11 +2056,12 @@ impl RemoteClient { if !auth_result { let reason = auth_error_msg .unwrap_or_else(|| "Authentication was rejected by the SSH target".to_string()); - error!(%reason, "Warpgate could not authenticate with SSH target"); + error!(?reason, "Warpgate could not authenticate with SSH target"); + let reason = reason.clone(); let _ = session .disconnect(russh::Disconnect::ByApplication, "", "") .await; - return Err(ConnectionError::Authentication); + return Err(ConnectionError::Authentication(reason)); } Ok(()) @@ -1169,7 +2298,55 @@ mod tests { use uuid::Uuid; - use super::resolve_chain_ids; + use super::{HopRole, resolve_chain_ids, role}; + + /// A host key check must go no further than the hop it was asked about. + /// + /// Moved here from the integration suite on §8's terms, and the terms were + /// met rather than assumed: the guard was pinned by + /// `test_the_host_key_check_reports_the_target_and_not_the_jump_host`, + /// which asks *whose key is reported* — a different question — and the + /// verifier measured that it passes with the guard disabled. An + /// end-to-end run cannot separate "stopped after the key" from "carried on + /// and the target refused us anyway", because a real sshd refuses on its + /// own. Here there is no target to do the refusing. + /// + /// What the guard prevents is a *check* becoming a session: the admin + /// endpoint asks for a host key, and answering it must not leave Warpgate + /// authenticated to the target. + #[test] + fn a_host_key_check_stops_at_the_hop_it_asked_about() { + assert!(HopRole::CheckedHost.stops_after_host_key()); + + // Everything else carries on. A hop merely traversed on the way has + // more chain behind it, and an ordinary connection is not a check at + // all. + assert!(!HopRole::TraversedWhileChecking.stops_after_host_key()); + assert!(!HopRole::Connecting.stops_after_host_key()); + } + + /// The role is decided by identity, and the two predicates follow from it. + /// + /// Pinned together because they are one decision read two ways: the hop the + /// caller named reports its key and stops; a hop on the way reports + /// nothing and continues; a plain connection reports every key and never + /// stops. + #[test] + fn each_role_reports_and_stops_as_its_name_says() { + let (asked_about, other) = (Uuid::new_v4(), Uuid::new_v4()); + + let checked = role(Some(asked_about), asked_about); + assert_eq!(checked, HopRole::CheckedHost); + assert!(checked.reports_host_key() && checked.stops_after_host_key()); + + let traversed = role(Some(asked_about), other); + assert_eq!(traversed, HopRole::TraversedWhileChecking); + assert!(!traversed.reports_host_key() && !traversed.stops_after_host_key()); + + let connecting = role(None, other); + assert_eq!(connecting, HopRole::Connecting); + assert!(connecting.reports_host_key() && !connecting.stops_after_host_key()); + } #[test] fn resolve_chain_ids_returns_ordered_chain() { @@ -1194,4 +2371,821 @@ mod tests { let jumps: HashMap> = HashMap::from([(a, Some(b))]); assert!(resolve_chain_ids(a, |id| jumps.get(&id).copied()).is_err()); } + + mod certificate_lifetime { + //! The validity window, at the level where every case can be reached. + //! + //! The integration test for a never-expiring certificate is skipped — + //! that input holds the session open for a reason not yet isolated — and + //! for two rounds the guard therefore had no coverage anywhere, while + //! the mutation matrix reported it covered because its anchor had gone + //! stale. Two independent instruments agreeing on an answer neither one + //! had measured. These do not need a session at all. + + use std::time::{Duration, SystemTime, UNIX_EPOCH}; + + use russh::keys::ssh_key::certificate::{Builder, CertType}; + use russh::keys::{Algorithm, PrivateKey, PublicKey}; + use warpgate_common::helpers::rng::get_crypto_rng; + + use crate::client::{MAX_CERTIFICATE_LIFETIME, certificate_mismatch}; + + const PRINCIPAL: &str = "root"; + const KEY_ID: &str = "warpgate:alice:0e6d1f4c"; + + fn now() -> u64 { + SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock is before the epoch") + .as_secs() + } + + /// A certificate over a freshly generated key, signed by a throwaway CA. + fn issued( + valid_after: u64, + valid_before: u64, + ) -> (russh::keys::ssh_key::Certificate, PublicKey) { + let ca = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating a CA key"); + let subject = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating a subject key"); + let mut builder = Builder::new_with_random_nonce( + &mut get_crypto_rng(), + subject.public_key(), + valid_after, + valid_before, + ) + .expect("building the certificate"); + builder.cert_type(CertType::User).expect("cert type"); + builder.key_id(KEY_ID).expect("key id"); + builder.valid_principal(PRINCIPAL).expect("principal"); + let certificate = builder.sign(&ca).expect("signing"); + (certificate, subject.public_key().clone()) + } + + fn verdict(valid_after: u64, valid_before: u64) -> Option { + let (certificate, key) = issued(valid_after, valid_before); + certificate_mismatch(&certificate, &key, PRINCIPAL, KEY_ID, &[], &[], None) + } + + /// The one response-side property with no check at all until now: every + /// other asks whether the certificate matches the request, and none + /// asked who signed it. The CA here is generated per call, so a pin + /// naming any other key must refuse — and an unparseable pin must refuse + /// too, rather than quietly checking nothing. + #[test] + fn a_certificate_from_an_unpinned_ca_is_refused() { + let ca = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating a CA key"); + let subject = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating a subject key"); + let mut builder = Builder::new_with_random_nonce( + &mut get_crypto_rng(), + subject.public_key(), + now() - 60, + now() + 600, + ) + .expect("building the certificate"); + builder.cert_type(CertType::User).expect("cert type"); + builder.key_id(KEY_ID).expect("key id"); + builder.valid_principal(PRINCIPAL).expect("principal"); + let certificate = builder.sign(&ca).expect("signing"); + + let signer = ca + .public_key() + .to_openssh() + .expect("serialising the signing CA"); + let elsewhere = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating another CA key") + .public_key() + .to_openssh() + .expect("serialising the other CA key"); + + assert!( + crate::client::certificate_signer_mismatch(&certificate, None).is_none(), + "no pin was configured, so nothing should have been refused" + ); + // Both directions. A check that refuses everything passes the + // negative case alone, and would break every session. + assert!( + crate::client::certificate_signer_mismatch(&certificate, Some(&signer)).is_none(), + "the certificate's own signing CA was refused as unpinned" + ); + assert!( + crate::client::certificate_signer_mismatch(&certificate, Some(&elsewhere)) + .is_some(), + "a certificate signed by a CA other than the pinned one was accepted" + ); + assert!( + crate::client::certificate_signer_mismatch(&certificate, Some("not-a-key")) + .is_some(), + "an unparseable pin silently checked nothing" + ); + } + + fn verdict_against(valid_before: u64, requested: Option) -> Option { + let (certificate, key) = issued(now() - 60, valid_before); + certificate_mismatch(&certificate, &key, PRINCIPAL, KEY_ID, &[], &[], requested) + } + + /// The operator's own number, which was sent and then never looked at. + /// Twenty-three hours is under the ceiling, so nothing refused it. + #[test] + fn a_certificate_longer_than_the_requested_ttl_is_refused() { + let reason = verdict_against(now() + 23 * 3600, Some(Duration::from_secs(90))) + .expect("a certificate far longer than requested"); + assert!( + reason.contains("longer than the 90s this target asked for"), + "refused for the wrong reason: {reason}" + ); + } + + #[test] + fn a_certificate_matching_the_requested_ttl_is_accepted() { + assert_eq!( + verdict_against(now() + 180, Some(Duration::from_secs(180))), + None + ); + } + + /// Clocks disagree; the issuer being generous is a different thing. + #[test] + fn a_certificate_within_the_skew_allowance_is_accepted() { + assert_eq!( + verdict_against(now() + 180 + 30, Some(Duration::from_secs(180))), + None + ); + } + + #[test] + fn a_certificate_past_the_skew_allowance_is_refused() { + assert!( + verdict_against(now() + 180 + 120, Some(Duration::from_secs(180))).is_some(), + "two minutes over a three-minute request should be refused" + ); + } + + /// With no TTL configured the role's own decides, and there is nothing + /// to compare against — only the ceiling applies. + #[test] + fn without_a_requested_ttl_only_the_ceiling_applies() { + assert_eq!(verdict_against(now() + 23 * 3600, None), None); + } + + #[test] + fn a_short_lived_certificate_is_accepted() { + assert_eq!(verdict(now() - 60, now() + 300), None); + } + + /// `u64::MAX` is OpenSSH's "never expires" sentinel — what + /// `ssh-keygen -V always:forever` and a Vault role with no TTL write. + /// + /// Writing this test is what established that the comment beside the + /// check was wrong: it claimed the sentinel makes `valid_before_time()` + /// return `None`. It does not — `ssh-key` reports it as a real instant + /// capped at `i64::MAX`, so the certificate was refused by the + /// maximum-lifetime arm, under the message "valid for 2562047787518949 + /// hours". Refused either way, but naming the wrong problem to whoever + /// has to act on it. + #[test] + fn a_never_expiring_certificate_is_refused() { + let reason = verdict(now() - 60, u64::MAX).expect("a never-expiring certificate"); + assert!( + reason.contains("never expires"), + "refused for the wrong reason: {reason}" + ); + } + + /// Above `i64::MAX` but not the sentinel: no tool writes this, an + /// issuer that wants the certificate not to expire can. This is the + /// only input that actually reaches the `None` arm. + #[test] + fn an_unrepresentable_expiry_is_refused() { + let reason = verdict(now() - 60, u64::MAX - 1).expect("an unrepresentable expiry"); + assert!( + reason.contains("unrepresentable"), + "refused for the wrong reason: {reason}" + ); + } + + #[test] + fn an_already_expired_certificate_is_refused() { + let reason = verdict(now() - 600, now() - 300).expect("an expired certificate"); + assert!( + reason.contains("already expired"), + "refused for the wrong reason: {reason}" + ); + } + + #[test] + fn a_certificate_outliving_the_bound_is_refused() { + let over = MAX_CERTIFICATE_LIFETIME + Duration::from_secs(3600); + let reason = + verdict(now() - 60, now() + over.as_secs()).expect("an overlong certificate"); + assert!( + reason.contains("far longer"), + "refused for the wrong reason: {reason}" + ); + } + + /// The boundary itself, so that tightening the bound cannot silently + /// start refusing what it is documented to allow. + #[test] + fn a_certificate_at_the_bound_is_accepted() { + assert_eq!( + verdict(now() - 60, now() + MAX_CERTIFICATE_LIFETIME.as_secs() - 60), + None + ); + } + } + + mod critical_options { + //! Pinning a value is what makes an option mandatory. + //! + //! Every case is here rather than end to end because the target's own + //! sshd refuses options it does not recognise, so an integration test + //! that watches for a failed connection cannot tell our refusal from + //! the target's. At this level the target is not in the picture at all. + + use std::time::{SystemTime, UNIX_EPOCH}; + + use russh::keys::ssh_key::certificate::{Builder, CertType}; + use russh::keys::{Algorithm, PrivateKey, PublicKey}; + use warpgate_common::SshCertificateCriticalOption; + use warpgate_common::helpers::rng::get_crypto_rng; + + use crate::client::certificate_mismatch; + + const PRINCIPAL: &str = "root"; + const KEY_ID: &str = "warpgate:alice:0e6d1f4c"; + + fn carrying(options: &[(&str, &str)]) -> (russh::keys::ssh_key::Certificate, PublicKey) { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock is before the epoch") + .as_secs(); + let ca = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating a CA key"); + let subject = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating a subject key"); + let mut builder = Builder::new_with_random_nonce( + &mut get_crypto_rng(), + subject.public_key(), + now - 60, + now + 300, + ) + .expect("building the certificate"); + builder.cert_type(CertType::User).expect("cert type"); + builder.key_id(KEY_ID).expect("key id"); + builder.valid_principal(PRINCIPAL).expect("principal"); + for (name, value) in options { + builder + .critical_option(*name, *value) + .expect("critical option"); + } + let certificate = builder.sign(&ca).expect("signing"); + (certificate, subject.public_key().clone()) + } + + fn carrying_extensions( + extensions: &[(&str, &str)], + ) -> (russh::keys::ssh_key::Certificate, PublicKey) { + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("clock is before the epoch") + .as_secs(); + let ca = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating a CA key"); + let subject = PrivateKey::random(&mut get_crypto_rng(), Algorithm::Ed25519) + .expect("generating a subject key"); + let mut builder = Builder::new_with_random_nonce( + &mut get_crypto_rng(), + subject.public_key(), + now - 60, + now + 300, + ) + .expect("building the certificate"); + builder.cert_type(CertType::User).expect("cert type"); + builder.key_id(KEY_ID).expect("key id"); + builder.valid_principal(PRINCIPAL).expect("principal"); + for (name, value) in extensions { + builder.extension(*name, *value).expect("extension"); + } + let certificate = builder.sign(&ca).expect("signing"); + (certificate, subject.public_key().clone()) + } + + fn pinned(name: &str, value: Option<&str>) -> SshCertificateCriticalOption { + SshCertificateCriticalOption { + name: name.to_owned(), + value: value.map(str::to_owned), + } + } + + fn verdict( + carried: &[(&str, &str)], + configured: &[SshCertificateCriticalOption], + ) -> Option { + let (certificate, key) = carrying(carried); + certificate_mismatch(&certificate, &key, PRINCIPAL, KEY_ID, configured, &[], None) + } + + /// A duplicate name must not disable the pin. + /// + /// `.find()` took whichever entry the operator typed first, so a config + /// naming an option twice — once bare, once pinned — passed the + /// mandatory-presence check on the pinned entry and then matched the + /// bare one, accepting any value at all while the admin UI showed a + /// pin. The order of the two rows decided whether the target was + /// confined, and nothing on screen said so. + #[test] + fn a_bare_duplicate_does_not_cancel_a_pinned_value() { + for configured in [ + vec![ + pinned("force-command", None), + pinned("force-command", Some("/usr/bin/backup")), + ], + // Both orders, because the defect was entirely about order. + vec![ + pinned("force-command", Some("/usr/bin/backup")), + pinned("force-command", None), + ], + ] { + let reason = verdict(&[("force-command", "/bin/sh")], &configured) + .expect("a certificate whose command is not the pinned one"); + assert!( + reason.contains("does not match the value"), + "refused for the wrong reason: {reason}" + ); + } + } + + /// Two pins that disagree cannot both be satisfied, so nothing may be. + #[test] + fn conflicting_pins_refuse_everything_rather_than_picking_one() { + let configured = [ + pinned("force-command", Some("/usr/bin/backup")), + pinned("force-command", Some("/usr/bin/restore")), + ]; + for offered in ["/usr/bin/backup", "/usr/bin/restore", "/bin/sh"] { + assert!( + verdict(&[("force-command", offered)], &configured).is_some(), + "{offered} satisfied a pair of pins that disagree" + ); + } + } + + /// The removal attack. Someone with role-write but no right to sign does + /// not have to add anything — taking the pinned `force-command` out + /// turns a target locked to one command into a shell. + #[test] + fn a_pinned_option_missing_from_the_certificate_is_refused() { + let reason = verdict(&[], &[pinned("force-command", Some("/usr/bin/backup"))]) + .expect("a certificate without the pinned option"); + assert!( + reason.contains("without the critical option") && reason.contains("force-command"), + "refused for the wrong reason: {reason}" + ); + } + + #[test] + fn a_pinned_option_with_the_wrong_value_is_refused() { + let reason = verdict( + &[("force-command", "/bin/sh")], + &[pinned("force-command", Some("/usr/bin/backup"))], + ) + .expect("a certificate with the wrong value"); + assert!( + reason.contains("does not match the value"), + "refused for the wrong reason: {reason}" + ); + } + + #[test] + fn a_pinned_option_with_the_configured_value_is_accepted() { + assert_eq!( + verdict( + &[("force-command", "/usr/bin/backup")], + &[pinned("force-command", Some("/usr/bin/backup"))] + ), + None + ); + } + + /// A bare name permits without requiring — the configuration an + /// all-mandatory list could not express, for a role that sets an option + /// only sometimes. + #[test] + fn an_option_permitted_by_name_may_be_absent() { + assert_eq!(verdict(&[], &[pinned("source-address", None)]), None); + } + + #[test] + fn an_option_permitted_by_name_accepts_any_value() { + assert_eq!( + verdict( + &[("source-address", "10.0.0.0/8")], + &[pinned("source-address", None)] + ), + None + ); + } + + /// The key ID is three colon-separated fields, so a colon in the + /// username shifts every one of them — and what reads that log then + /// names the wrong person, which is the single claim this feature + /// makes. The admin API refuses one now; a name from an IdP never + /// passes through it, so the structure is held where it is built. + #[test] + fn a_username_carrying_a_colon_cannot_shift_the_key_id_fields() { + // Through the shipped function, not a copy of it written here. + let key_id = format!( + "warpgate:{}:0e6d1f4c-0000-0000-0000-000000000000", + crate::client::key_id_field("root:admin") + ); + let fields: Vec<&str> = key_id.split(':').collect(); + assert_eq!( + fields.len(), + 3, + "the key ID split into {} fields", + fields.len() + ); + assert_eq!(fields[1], "root%3Aadmin"); + } + + /// The field count was the only thing checked, and it is satisfied by a + /// substitution that maps two different people onto one name. Raised + /// externally: `root:admin` and `root_admin` both used to read as + /// `root_admin` in the target's sshd log and in Vault's audit log. + #[test] + fn two_usernames_cannot_collide_in_a_key_id() { + assert_ne!( + crate::client::key_id_field("root:admin"), + crate::client::key_id_field("root_admin"), + "two different usernames produced one key ID field" + ); + // The encoding has to survive a name that already looks encoded, + // or it is a substitution again one level up. + assert_ne!( + crate::client::key_id_field("root:admin"), + crate::client::key_id_field("root%3Aadmin"), + "an encoded name and a literal one produced one key ID field" + ); + } + + /// `UNATTRIBUTED` stands in this field when no user is recorded. A user + /// of that name read identically, which is the same defect as the + /// attribution one beside it and was missed because the reserved names + /// lived in two places. + #[test] + fn a_username_cannot_impersonate_the_unattributed_placeholder() { + assert_ne!( + crate::client::user_key_id_field(crate::client::UNATTRIBUTED), + crate::client::UNATTRIBUTED, + "a user named after the placeholder was left as the placeholder" + ); + } + + #[test] + fn an_ordinary_username_is_left_alone() { + assert_eq!(crate::client::key_id_field("alice"), "alice"); + } + + /// `attribution()` puts `admin-token` in this field when the admin API + /// token drives a session. A user of that name produced the same three + /// fields, in the target's sshd log and in Vault's audit log — the two + /// records this feature exists to make trustworthy. The admin API + /// refuses the name; SSO auto-provisioning inserts the IdP's claim + /// without asking it. + #[test] + fn a_username_cannot_impersonate_the_gateways_own_attribution() { + for reserved in warpgate_common_http::auth::TOKEN_ATTRIBUTIONS { + assert_ne!( + crate::client::user_key_id_field(reserved), + reserved, + "a user named {reserved} produces the attribution's own key ID" + ); + } + } + + /// And the other direction, which the first version of this fix did not + /// have. The gateway's own attribution must pass through unchanged: + /// substituting it renames the thing it identifies, and two guards + /// caught that by failing their baseline — the key ID had become + /// `warpgate:admin-token_:`. + #[test] + fn the_gateways_own_attribution_is_left_alone() { + for reserved in warpgate_common_http::auth::TOKEN_ATTRIBUTIONS { + assert_eq!( + crate::client::key_id_field(reserved), + reserved, + "the gateway's own attribution was renamed on its way to the key ID" + ); + } + } + + /// A chain that does not contain the host being asked about cannot + /// answer the question, and used to be walked to the end anyway — + /// every hop traversed, no key reported, a live session handed back. + /// The property the integration test was believed to hold and does + /// not: measured twice, the stalling fixture never reaches this code. + /// It delivers the host key on the wire, but russh does not call + /// `check_server_key` until the key exchange completes, and the fixture + /// mutes before `NEWKEYS`. Letting `NEWKEYS` through instead trips + /// strict-kex and the client disconnects in three seconds. So what that + /// test measures is the plain handshake bound, which has its own guard. + /// + /// What is left to state is the property itself: whatever the pause is, + /// the answer has to bring the bound back to the target's own, and not + /// to something a stalled target can sit inside. Pushing the resume out + /// instead of back is the exact regression this exists to catch, and it + /// is what the previous version of this fix did. + #[tokio::test] + async fn answering_a_host_key_question_puts_the_targets_own_bound_back() { + use std::time::Duration; + + use crate::client::{ + HANDSHAKE_TIMEOUT, pause_for_host_key_question, resume_after_host_key_answer, + }; + + // The real `Sleep` the connect loop holds, moved by the real + // functions the connect loop calls. The previous version of this + // test compared two constants and called neither — it would have + // passed with a call site deleted, or with the two durations + // swapped between them, which is the regression it exists to catch. + let deadline = tokio::time::sleep(HANDSHAKE_TIMEOUT); + tokio::pin!(deadline); + let armed = deadline.deadline(); + + pause_for_host_key_question(deadline.as_mut()); + let paused = deadline.deadline(); + assert!( + paused > armed + Duration::from_secs(24 * 60 * 60), + "the pause did not put the bound beyond any time a person spends \ + reading a fingerprint" + ); + + resume_after_host_key_answer(deadline.as_mut()); + let resumed = deadline.deadline(); + assert!( + resumed < paused, + "the answer left the connection on the pause, so a target that \ + goes quiet after offering a host key is never given up on" + ); + assert!( + resumed <= tokio::time::Instant::now() + HANDSHAKE_TIMEOUT, + "the answer left the connection on a bound longer than the \ + target's own" + ); + } + + #[test] + fn a_chain_without_the_host_asked_about_cannot_answer() { + let asked_about = uuid::Uuid::new_v4(); + let other = uuid::Uuid::new_v4(); + + assert!(crate::client::chain_can_answer(None, &[other])); + assert!(crate::client::chain_can_answer( + Some(asked_about), + &[other, asked_about] + )); + assert!( + !crate::client::chain_can_answer(Some(asked_about), &[other]), + "a chain missing the host asked about reported that it could answer" + ); + } + + /// The budget for authentication is not the budget for the transport + /// handshake, and used to be. + /// + /// Five calls at the default 10s `vault.timeout` is fifty seconds + /// A certificate marked never-expiring reports a `valid_before` beyond + /// the year 9999, and `humantime`'s `Display` returns `Err` there + /// rather than truncating — so `to_string()` panicked, in a tokio + /// worker, while building the message that describes the window. The + /// check that refuses such a certificate sits *after* that line and + /// never ran; the client was left holding a connection nobody would + /// answer, which is what the integration test had been parked on for a + /// week as "holds the session open for a reason not yet isolated". + #[test] + fn a_far_future_expiry_is_described_rather_than_panicked_on() { + use std::time::{Duration, UNIX_EPOCH}; + + use crate::client::describe_certificate_time; + + // The first second of the year 9999, where `humantime` gives up. + let past_rendering = UNIX_EPOCH + Duration::from_secs(253_402_300_800); + + let described = describe_certificate_time(Some(past_rendering)); + assert!( + !described.is_empty(), + "a time past what can be rendered produced nothing" + ); + assert_eq!(describe_certificate_time(None), "unbounded"); + // And the ordinary case still renders as a date, or this would pass + // by describing everything as unrenderable. + assert!( + describe_certificate_time(Some(UNIX_EPOCH + Duration::from_secs(1_700_000_000))) + .starts_with("2023-"), + "an ordinary expiry stopped rendering as a date" + ); + } + + /// The sanitiser had no test of its own, and the nearest one that + /// looked like coverage asserts a string that also appears in an + /// unrelated variant's `Display` — so it passes identically with the + /// sanitising removed. Raised externally. + #[test] + fn no_internal_error_text_reaches_a_client_message() { + use warpgate_common::WarpgateError; + + use crate::ConnectionError; + + let internals = "relation warpgate_user column password_hash"; + let shown = ConnectionError::Warpgate(WarpgateError::other(std::io::Error::other( + internals, + ))) + .client_message(); + + assert!( + !shown.contains(internals), + "internal error text reached a client message: {shown}" + ); + } + + /// An operator checking a host key gets one screen for two entirely + /// different jobs: "I cannot reach this host" and "this host's key is + /// not trusted" both rendered as `SSH protocol error`. Raised + /// externally, out of chasing a sandbox failure that read as the second + /// while being the first. + #[test] + fn an_unreachable_target_does_not_read_like_an_untrusted_key() { + use crate::ConnectionError; + + let unreachable = ConnectionError::Ssh(russh::Error::IO(std::io::Error::from( + std::io::ErrorKind::ConnectionRefused, + ))) + .client_message(); + + assert_ne!( + unreachable, + ConnectionError::UntrustedJumpHost.client_message(), + "unreachable and untrusted render identically" + ); + assert!( + !unreachable.contains("protocol error"), + "an unreachable target still reads as a protocol error: {unreachable}" + ); + // The refusal is named, not just the category — an operator who + // cannot tell "refused" from "no route" has to go and find out. + assert!( + unreachable.contains("refused"), + "the reason was flattened away: {unreachable}" + ); + } + + /// A target that takes the certificate and then says nothing is given up + /// on for its own reasons. + /// + /// The budget above has to cover the issuer as well, and for a + /// certificate target it grows with `vault.timeout`, which config does + /// not clamp from above. Sharing it meant a silent target held the + /// session, the ephemeral private key and a live certificate for a + /// window measured in Vault's slowness — 55 seconds by default and + /// unbounded in principle. Driven on a paused clock, so it costs no + /// wall time and cannot go flaky on a loaded machine. + #[tokio::test] + async fn a_target_that_never_answers_userauth_is_given_up_on() { + use std::time::Duration; + + use crate::ConnectionError; + use crate::client::bounded_userauth_within; + + let bound = Duration::from_millis(50); + let slower_than_its_own_bound = async { + tokio::time::sleep(bound * 4).await; + Ok(()) + }; + + assert!( + matches!( + bounded_userauth_within(bound, slower_than_its_own_bound).await, + Err(ConnectionError::TargetAuthenticationTimeout) + ), + "a target that never answered was waited on past its own bound" + ); + } + + /// against a thirty-second bound, so a Vault that was slow but working + /// timed out and the message named the target. + #[test] + fn a_certificate_target_gets_a_budget_that_fits_its_vault_calls() { + use std::time::Duration; + + use warpgate_common::{SSHTargetAuth, SshTargetCertificateAuth, SshTargetPasswordAuth}; + + use crate::client::{AUTHENTICATION_TIMEOUT, authentication_budget}; + + let certificate = SSHTargetAuth::Certificate(SshTargetCertificateAuth::default()); + + // The default: five 10s calls do not fit in 30s, so the budget grows. + let budget = authentication_budget(&certificate, Some(Duration::from_secs(10))); + assert!( + budget >= Duration::from_secs(50), + "five 10s calls do not fit in {budget:?}" + ); + + // A fast Vault does not shrink it below the floor. + assert_eq!( + authentication_budget(&certificate, Some(Duration::from_secs(1))), + AUTHENTICATION_TIMEOUT + ); + + // Nothing to budget for when the target does not use an issuer. + let password = SSHTargetAuth::Password(SshTargetPasswordAuth { + password: String::new().into(), + }); + assert_eq!( + authentication_budget(&password, Some(Duration::from_secs(10))), + AUTHENTICATION_TIMEOUT + ); + assert_eq!( + authentication_budget(&certificate, None), + AUTHENTICATION_TIMEOUT + ); + } + + /// Extensions decide what a session can *do*, and were logged and + /// nothing else. `force-command` governs the shell and exec channels; + /// OpenSSH opens `direct-tcpip` and reaches the user's agent on the + /// strength of the certificate alone. + #[test] + fn an_extension_the_target_did_not_name_is_refused() { + let (certificate, key) = carrying_extensions(&[("permit-port-forwarding", "")]); + let reason = certificate_mismatch( + &certificate, + &key, + PRINCIPAL, + KEY_ID, + &[], + &["permit-pty".to_owned()], + None, + ) + .expect("an unexpected extension"); + assert!( + reason.contains("permit-port-forwarding") && reason.contains("does not allow"), + "refused for the wrong reason: {reason}" + ); + } + + #[test] + fn a_named_extension_is_accepted() { + let (certificate, key) = carrying_extensions(&[("permit-pty", "")]); + assert_eq!( + certificate_mismatch( + &certificate, + &key, + PRINCIPAL, + KEY_ID, + &[], + &["permit-pty".to_owned()], + None, + ), + None + ); + } + + /// An absent extension grants nothing, so unlike a pinned critical + /// option there is nothing to remove that would widen access. + #[test] + fn a_named_extension_may_be_absent() { + let (certificate, key) = carrying_extensions(&[]); + assert_eq!( + certificate_mismatch( + &certificate, + &key, + PRINCIPAL, + KEY_ID, + &[], + &[ + "permit-pty".to_owned(), + "permit-agent-forwarding".to_owned() + ], + None, + ), + None + ); + } + + /// The other direction, unchanged: nothing unlisted gets through. + #[test] + fn an_option_the_target_did_not_name_is_refused() { + let reason = verdict(&[("force-command", "/bin/sh")], &[]) + .expect("an unexpected critical option"); + assert!( + reason.contains("does not allow"), + "refused for the wrong reason: {reason}" + ); + } + } } diff --git a/warpgate-protocol-ssh/src/server/service_output.rs b/warpgate-protocol-ssh/src/server/service_output.rs index aaa035a92..426a576af 100644 --- a/warpgate-protocol-ssh/src/server/service_output.rs +++ b/warpgate-protocol-ssh/src/server/service_output.rs @@ -34,13 +34,61 @@ pub enum VisualConnectionChainItem { Link { text: String, url: String }, } +/// Strips control characters from a string on its way to a terminal. +/// +/// The chain is drawn from target *names*, and a name is free text an operator +/// types into the admin UI. Written straight out, a name containing `\x1b[2J` +/// clears the screen of every user who connects through that target, and one +/// containing an OSC-8 sequence draws a hyperlink pointing wherever it likes — +/// in the frame Warpgate itself is printing, which is where a user has most +/// reason to trust what they see. +/// +/// This is the same bug class as the certificate-derived text in the client, +/// found by going back over the other places that write toward a PTY once the +/// class had been named there. Lower severity: a target name needs +/// `TargetsEdit`, so this is not a privilege boundary, and the audience is +/// whoever connects rather than whoever configures. Same fix regardless. +fn without_control_characters(text: &str) -> Cow<'_, str> { + if text.chars().any(char::is_control) { + Cow::Owned(text.chars().filter(|c| !c.is_control()).collect()) + } else { + Cow::Borrowed(text) + } +} + +/// The same, but a line break survives. +/// +/// This is the form the PTY sinks use. `emit_service_message` turns `\n` into +/// `\r\n` itself, so stripping every control character there would silently +/// join the lines of any multi-line message instead of escaping anything. +/// +/// Separate from `without_control_characters` rather than a flag on it: the +/// chain renderer must not keep newlines — a name with one in it would break the +/// drawing — and a shared function with a boolean would eventually be called +/// with the wrong one. +pub(super) fn without_control_characters_except_newline(text: &str) -> Cow<'_, str> { + if text.chars().any(|c| c.is_control() && c != '\n') { + Cow::Owned( + text.chars() + .filter(|c| !c.is_control() || *c == '\n') + .collect(), + ) + } else { + Cow::Borrowed(text) + } +} + impl VisualConnectionChainItem { pub fn ansi(&self) -> Cow<'_, str> { match self { - Self::Text(s) => Cow::Borrowed(s), - Self::Link { text, url } => { - Cow::Owned(format!("\x1b]8;;{url}\x1b\\{text}\x1b]8;;\x1b\\")) - } + Self::Text(s) => without_control_characters(s), + // The escapes here are Warpgate's own, wrapped around text and a URL + // that are not. + Self::Link { text, url } => Cow::Owned(format!( + "\x1b]8;;{}\x1b\\{}\x1b]8;;\x1b\\", + without_control_characters(url), + without_control_characters(text) + )), } } } @@ -259,3 +307,38 @@ impl Drop for ServiceOutput { tokio::spawn(async move { signal.send(()).await }); } } + +#[cfg(test)] +mod tests { + use super::VisualConnectionChainItem; + + /// A target name is free text an operator types; it is drawn into the PTY of + /// everyone who connects through that target. + #[test] + fn a_target_name_cannot_write_escape_sequences_to_the_terminal() { + let hostile = VisualConnectionChainItem::Text("prod\x1b[2J\x1b[H".to_owned()); + let rendered = hostile.ansi(); + assert_eq!(rendered, "prod[2J[H"); + assert!(!rendered.contains('\x1b'), "{rendered:?}"); + } + + /// Asserted as a property rather than a literal: the wrapper emits four + /// escapes of its own, and the data must contribute none. Writing the + /// expected bytes out by hand got this wrong in a way that looked like a + /// code bug — the surviving backslash is inert, because the `ESC` that + /// would have made it an OSC terminator is gone. + #[test] + fn a_link_keeps_its_own_escapes_but_not_the_data_s() { + let item = VisualConnectionChainItem::Link { + text: "Warp\x1bgate".to_owned(), + url: "https://example.com/\x1b]8;;evil\x1b\\".to_owned(), + }; + let rendered = item.ansi(); + assert_eq!( + rendered.matches('\x1b').count(), + 4, + "the data contributed an escape: {rendered:?}" + ); + assert!(rendered.contains("Warpgate"), "{rendered:?}"); + } +} diff --git a/warpgate-protocol-ssh/src/server/session.rs b/warpgate-protocol-ssh/src/server/session.rs index a43aabe75..8dfc79713 100644 --- a/warpgate-protocol-ssh/src/server/session.rs +++ b/warpgate-protocol-ssh/src/server/session.rs @@ -42,12 +42,14 @@ use super::service_output::ServiceOutput; use super::session_handle::SessionHandleCommand; use crate::compat::ContextExt; use crate::server::get_allowed_auth_methods; -use crate::server::service_output::{VisualConnectionChainItem, paint_fg}; +use crate::server::service_output::{ + VisualConnectionChainItem, paint_fg, without_control_characters_except_newline, +}; use crate::server::target_menu::{MenuEvent, spawn_target_menu_loop}; use crate::{ ChannelOperation, ConnectionError, DirectTCPIPParams, PtyRequest, RCCommand, RCCommandReply, RCEvent, RCState, RemoteClient, ResolvedSshChainHost, ServerChannelId, SshClientError, - SshRecordingMetadata, X11Request, resolve_ssh_chain, + SshRecordingMetadata, X11Request, client_error_message, resolve_ssh_chain, }; const EVENT_QUEUE_CAPACITY: usize = 128; @@ -466,8 +468,20 @@ impl ServerSession { Ok(()) } + /// Escaping happens here, at the sink, and not in the callers. + /// + /// It used to happen in the callers. Every round of review since found + /// another one that had been missed — a target name in one message, a + /// certificate's principals in another — because a fix applied at the point + /// of use only ever covers the points somebody went looking at. Both PTY + /// sinks now escape unconditionally, so a new call site cannot reopen the + /// hole by forgetting, and Warpgate's own colour codes are added after the + /// text has been through it. pub async fn emit_service_message(&self, msg: &str) -> Result<()> { - debug!("Service message: {}", msg); + // Before the escaping below, not after: this logs the raw message, + // so a `\n` in a certificate's option name or a Vault error body forges + // a log record even though the same text reaches the terminal escaped. + debug!("Service message: {msg:?}"); let _ = self .emit_pty_output(self.service_output.erase_display().as_bytes()) @@ -475,7 +489,7 @@ impl ServerSession { let output = format!( "{} {}\r\n", paint_fg(Color::Blue, false, "● Warpgate:"), - msg.replace('\n', "\r\n") + without_control_characters_except_newline(msg).replace('\n', "\r\n") ); self.emit_pty_output(output.as_bytes()).await } @@ -487,6 +501,7 @@ impl ServerSession { .emit_pty_output(self.service_output.erase_display().as_bytes()) .await; } + let msg = without_control_characters_except_newline(msg).replace('\n', "\r\n"); let output = format!("{} {msg}\r\n", paint_fg(Color::Red, false, "● Warpgate:")); self.emit_pty_output(output.as_bytes()).await } @@ -546,10 +561,8 @@ impl ServerSession { let visual_chain = self.make_visual_connection_chain(&ssh_chain[..]).await?; self.rc_state = RCState::Connecting; - self.send_command(RCCommand::Connect( - ssh_chain.into_iter().map(|x| x.ssh_options).collect(), - )) - .map_err(|_| anyhow::anyhow!("cannot send command"))?; + self.send_command(RCCommand::Connect(ssh_chain)) + .map_err(|_| anyhow::anyhow!("cannot send command"))?; self.emit_pty_output(b"\r\n").await?; self.service_output.start_progress(visual_chain).await; Ok(()) @@ -1079,21 +1092,38 @@ impl ServerSession { ) .await?; } - ConnectionError::Authentication => { + ConnectionError::Authentication(ref reason) => { + // The reason is what tells a wrong credential apart from + // a target whose clock disagrees, which is the most + // common cause of a short-lived certificate being + // refused. It used to reach the server log and stop + // there. let _ = self - .emit_pty_error("SSH target rejected Warpgate's authentication request") + .emit_pty_error(&format!( + "SSH target rejected Warpgate's authentication request: {reason}" + )) .await; } error => { + tracing::error!(%error, "Target connection failed"); let _ = self - .emit_pty_error(&format!("Target connection failed: {error}")) + .emit_pty_error(&format!( + "Target connection failed: {}", + error.client_message() + )) .await; } } } RCEvent::Error(e) => { self.service_output.stop_progress(); - let _ = self.emit_pty_error(&format!("Error: {e}")).await; + // The full error to the log, a constant to the terminal. The + // detail is what an operator needs and what a connected user + // must not be handed; printing `{e}` gave it to the user and + // was the one path to this sink that no round of hardening had + // touched. + error!(error=%e, "Client session error"); + let _ = self.emit_pty_error(client_error_message(&e)).await; self.disconnect_server().await; } RCEvent::Output(channel, data) => { diff --git a/warpgate-vault/Cargo.toml b/warpgate-vault/Cargo.toml new file mode 100644 index 000000000..3cb9fe437 --- /dev/null +++ b/warpgate-vault/Cargo.toml @@ -0,0 +1,29 @@ +[package] +name = "warpgate-vault" +version = "0.27.4" +edition = "2024" +license = "Apache-2.0" +publish = false + +[dependencies] +data-encoding.workspace = true +reqwest = { workspace = true, features = ["json"] } +serde.workspace = true +serde_json.workspace = true +thiserror.workspace = true +tokio = { workspace = true, features = ["fs", "sync"] } +tracing.workspace = true +url.workspace = true +warpgate-aws = { path = "../warpgate-aws" } +warpgate-common = { path = "../warpgate-common" } +zeroize = "1.8" + +[dev-dependencies] +proptest = "1" +rcgen.workspace = true +rustls = { workspace = true, features = ["aws_lc_rs"] } +rustls-pki-types.workspace = true +tempfile = "3" +tokio = { workspace = true, features = ["net", "time"] } +tokio-rustls = { workspace = true, features = ["aws-lc-rs"] } + diff --git a/warpgate-vault/README.md b/warpgate-vault/README.md new file mode 100644 index 000000000..a2e2da2d3 --- /dev/null +++ b/warpgate-vault/README.md @@ -0,0 +1,456 @@ +# SSH certificate authentication for targets + +Warpgate can authenticate to an SSH target with a short-lived OpenSSH certificate +issued on demand by HashiCorp Vault, instead of a private key it stores. + +## What changes + +| | Stored key (`publickey`) | Vault certificate (`certificate`) | +|---|---|---| +| On the Warpgate host | a private key every target trusts, indefinitely | nothing a target would accept | +| On each target | an `authorized_keys` entry per gateway key | one `TrustedUserCAKeys` line, set once | +| Revoking access | edit `authorized_keys` on every host | stop issuing at Vault | +| Target's own audit log | one shared gateway identity | the Warpgate username and session | +| Adding a target | provision the key first | nothing | + +Warpgate generates an ed25519 keypair per connection, has Vault sign the public +half for the target's username, authenticates, and drops both. The private half +never reaches disk, the database, or a log. + +``` + ssh user:target@warpgate + │ + ▼ + ┌──────────┐ 1. workload identity ┌───────┐ + │ Warpgate │ ──────────────────────► │ Vault │ + │ │ ◄────────────────────── │ CA │ + └──────────┘ 2. certificate (2 min) └───────┘ + │ ▲ + │ 3. ephemeral key + certificate │ trusts + ▼ │ + ┌──────────┐ TrustedUserCAKeys ──────────┘ + │ target │ (no authorized_keys) + └──────────┘ +``` + +Vault is on the critical path of every session — see *Operational notes* below. + +## 1. Vault: the SSH CA + +```bash +vault secrets enable -path=ssh-client-signer ssh +vault write ssh-client-signer/config/ca generate_signing_key=true key_type=ed25519 + +vault write ssh-client-signer/roles/warpgate - <<'EOF' +{ + "key_type": "ca", + "allow_user_certificates": true, + "allow_user_key_ids": true, + "allowed_users": "deploy,app", + "default_user": "deploy", + "ttl": "2m", + "max_ttl": "5m", + "default_extensions": { "permit-pty": "" } +} +EOF + +vault policy write warpgate - <<'EOF' +path "ssh-client-signer/sign/warpgate" { + capabilities = ["update"] +} +EOF +``` + +Two settings deserve attention: + +- **`allowed_users`** is the real access boundary. Never `*`. Targets of differing + privilege belong to differing roles, selected per target in Warpgate. +- **`allow_user_key_ids`** must be `true`. Without it Vault rejects the request + outright, and every session would be attributed to the Warpgate service + identity instead of to a person. +- **A target with no username** asks for a certificate naming the *connecting + Warpgate user* instead — the same substitution every other SSH auth method + makes. That is not a bypass, since `allowed_users` still bounds what Vault will + issue, but it means the role must list every Warpgate username that may reach + the target. Setting the username explicitly is almost always what you want. + +`default_extensions` grants the minimum. Add `permit-port-forwarding` or +`permit-agent-forwarding` only if your users need them. + +## 2. Warpgate's identity to Vault + +This decides whether the feature delivers what it promises. Every method reads +its credential from a file or a metadata service, never from `warpgate.yaml`. + +| Deployment | `kind` | What is on disk | +|---|---|---| +| Kubernetes | `kubernetes` | nothing durable — the kubelet mounts and rotates the token | +| EC2 | `aws` | nothing — the instance role's credentials are short-lived | +| GCE | `gcp` | nothing — the token comes from the metadata server | +| Azure VM | `azure` | nothing — the token comes from IMDS | +| Anything else | `app_role` | a short-lived secret ID, ideally response-wrapped | + +Common part of the config: + +```yaml +vault: + address: https://vault.internal:8200 + mount: ssh-client-signer # default + default_role: warpgate + timeout: 10s # default + certificate_ttl: 3m # optional; the role's TTL decides when unset + auth: + ... +``` + +`certificate_ttl` is a request, not a grant: Vault clamps it to the role's +`max_ttl`, so it can shorten the window but never widen it. It is there so the +lifetime can be tightened from Warpgate's side without an edit to Vault. Under a +second is refused at startup — both issuers reject it, and failing at config +load names the line instead of failing every session later. + +Whatever the role grants, Warpgate checks the certificate it gets back and +refuses to use it if the window is wrong. Four cases, because a role's `max_ttl` +is not something Warpgate can see: + +| Returned certificate | What happens | +|---|---| +| Valid for more than **24 hours** | Refused. Generous — a role would have to be badly misconfigured to exceed it — but this feature exists to hand the target a credential that is worthless minutes later | +| Valid for longer than the `certificate_ttl` that was asked for | Refused. Vault may shorten a request and never widen one, so a longer window back means the role is not the one intended. A few seconds of slack is allowed for the round trip | +| Marked never-expiring (`ssh-keygen -V always:forever`, or a role with no TTL) | Refused | +| Already expired | Refused, with a message naming the clock — the usual cause is skew on this host or the target, not a bad credential | + +Set `certificate_ttl` well below the 24-hour ceiling; it is a backstop against a +misconfigured role, not a target to aim at. + +Warpgate refuses a plain-HTTP `address` unless it is loopback. Vault's +certificate is verified against the host's trust store, plus `ca_bundle` if set: + +```yaml +vault: + ca_bundle: /etc/warpgate/vault-ca.pem # optional +``` + +The bundle is *added* to the host's roots rather than replacing them, and an +unreadable or malformed file fails at startup naming the path. Leave it unset and +install the CA on the host or in the container image instead +(`/usr/local/share/ca-certificates` + `update-ca-certificates` on Debian-based +images) — either works. + +There is deliberately no setting to skip verification. The HTTP and Kubernetes +target paths offer `verify: false` for devices whose certificates cannot be +fixed; there is no equivalent case for Vault, and the Vault token crosses this +connection in a header. + +### Kubernetes + +```yaml + auth: + kind: kubernetes + role: warpgate + # token_path defaults to the projected service account token +``` +```bash +vault auth enable kubernetes +vault write auth/kubernetes/config kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" +vault write auth/kubernetes/role/warpgate \ + bound_service_account_names=warpgate \ + bound_service_account_namespaces=warpgate \ + token_policies=warpgate ttl=1h +``` + +### AWS + +```yaml + auth: + kind: aws + role: warpgate + server_id: vault.internal # must match iam_server_id_header_value +``` +```bash +vault auth enable aws +vault write auth/aws/config/client iam_server_id_header_value=vault.internal +vault write auth/aws/role/warpgate auth_type=iam \ + bound_iam_principal_arn="arn:aws:iam:::role/" \ + token_policies=warpgate ttl=1h +``` + +Warpgate signs an `sts:GetCallerIdentity` request; Vault replays it against STS +to learn who signed it. No credential is disclosed. + +**`server_id` is optional and leaving it out is the insecure choice.** It is +bound into the signature as `X-Vault-AWS-IAM-Server-ID`, and Vault compares it +against its own `iam_server_id_header_value`. Without it the signed request +proves only *this principal signed something* — so anyone who obtains one can +present it to any other Vault that trusts the same principal. Warpgate logs a +warning at startup when it is unset rather than refusing: Vault ignores the +header entirely unless `iam_server_id_header_value` is configured, so demanding +a value here would mean inventing one for a server that will not look at it. +Set both, or accept the warning knowingly. + +Leave `region` unset. Vault replays against the global endpoint, which rejects a +signature scoped to any other region. Set it only if Vault has a matching +`sts_endpoint`. + +### GCE + +```yaml + auth: + kind: gcp + role: warpgate +``` +```bash +gcloud services enable iam.googleapis.com # required, see Troubleshooting +vault auth enable gcp +vault write auth/gcp/config +vault write auth/gcp/role/warpgate type=gce \ + bound_projects= bound_zones= \ + token_policies=warpgate ttl=1h +``` + +### Azure + +```yaml + auth: + kind: azure + role: warpgate +``` +```bash +vault auth enable azure +vault write auth/azure/config \ + tenant_id= resource=https://management.azure.com/ \ + client_id= client_secret= +vault write auth/azure/role/warpgate \ + bound_subscription_ids= \ + bound_resource_groups= \ + bound_service_principal_ids= \ + token_policies=warpgate ttl=1h +``` + +The VM needs a system-assigned managed identity. Vault needs its own service +principal with `Reader` on the resource group to verify the VM through ARM. + +### AppRole + +```yaml + auth: + kind: app_role + role_id: + secret_id_path: /run/secrets/vault-secret-id +``` + +The secret ID is read fresh on every login, so it can be rotated underneath a +running Warpgate. + +## 3. Targets + +Add the CA public key and remove nothing else yet: + +```bash +vault read -field=public_key ssh-client-signer/config/ca | sudo tee /etc/ssh/trusted-ca.pub +sudo chmod 600 /etc/ssh/trusted-ca.pub +echo 'TrustedUserCAKeys /etc/ssh/trusted-ca.pub' | sudo tee -a /etc/ssh/sshd_config +sudo sshd -t && sudo systemctl reload sshd +``` + +The target now accepts certificates from this CA *in addition* to whatever it +accepted before. Once certificate auth is confirmed working, remove the gateway's +entry from `authorized_keys`. + +For a second boundary on the target side, `AuthorizedPrincipalsFile` restricts +which principals may log in as which local user. + +## 4. Warpgate target configuration + +In the admin UI, an SSH target's **Authenticate using** now offers *Certificate +issued by Vault*, with an optional signing role. The option appears only when +`vault:` is configured. Leave the role empty to use `default_role`. + +Through the API: + +```json +{ + "name": "prod-db", + "options": { + "kind": "Ssh", "host": "db.internal", "port": 22, "username": "deploy", + "auth": { "kind": "Certificate", "role": "warpgate-prod" } + } +} +``` + +## Verification + +```bash +# Warpgate: one login, one certificate per session +grep -E 'Authenticated to Vault|Issued an SSH certificate' + +# The target names the person, not the gateway +grep 'Accepted certificate ID' /var/log/auth.log + +# Nothing was persisted: only the bootstrap keys, whatever you do +# (admin UI → Config → SSH, or GET /@warpgate/admin/api/ssh/own-keys) +``` + +Expected in the target's log: + +``` +Accepted certificate ID "warpgate:alice:8fae5e3d-..." (serial ...) signed by ED25519 CA ... +``` + +## Troubleshooting + +**`SignatureDoesNotMatch: Credential should be scoped to a valid region`** (AWS) — +`region` is set in the config but Vault replays against the global STS endpoint. +Remove it, or set Vault's `sts_endpoint` to match. + +**`SERVICE_DISABLED ... Identity and Access Management (IAM) API`** (GCP) — Vault +resolves the service account named in the identity token through the IAM API. +`gcloud services enable iam.googleapis.com`, then wait a minute. + +**`expected specific bound_group_ids or bound_service_principal_ids`** (Azure) — +the Vault role binds only a subscription and resource group. Add +`bound_service_principal_ids`, the object ID of the VM's managed identity (the +`oid` claim of the IMDS token). + +**A policy change appears to have no effect** — narrower than it looks, because +Warpgate already recovers from most of it. Vault evaluates a policy's rules at +request time, so editing what a policy *allows* takes effect immediately for a +token already issued. And when signing comes back `403`, Warpgate drops the +cached token, logs in once more and retries within the same request — so a +revoked token, a resealed or restarted Vault, and a token whose lease Vault +stopped honouring all resolve on their own, and a denial that survives the retry +is a real one. + +What is left: changing which policies the *auth role* attaches applies only to +tokens issued after the change, and Warpgate keeps its token until shortly +before the lease runs out. If you have granted new permissions that way and +signing is not failing outright, restart Warpgate or wait out `token_ttl`. + +**`Vault issued a certificate valid for N hours, far longer than a session +credential should be`** — Warpgate refuses anything over 24 hours, whatever the +role's `max_ttl` allows, and the certificate was never offered to the target. +Shorten the role's `max_ttl`, or set `certificate_ttl` to hold the window down +from Warpgate's side. The companion messages are `never expires`, for a role with +no TTL at all, and `already expired`, which is almost always clock skew rather +than a bad credential. + +**`Certificate authentication was rejected by the SSH target`** — the certificate +was issued but the target refused it. Check `sshd -T | grep trustedusercakeys`, +that the principal matches the target username, and the clock skew between +Warpgate and the target. + +**`No Vault server is configured`** — the target uses certificate auth but +`vault:` is absent from the config. + +## OpenBao Compatibility + +Warpgate's Vault integration is fully compatible with **OpenBao**. The SSH secrets engine API and `/sign/` endpoints are identical. + +This is asserted rather than assumed: `tests/test_vault_contract.py` runs the +same suite against `hashicorp/vault` and `openbao/openbao`, and every claim the +test stub makes about server behaviour is pinned against both. + +Note for OpenBao deployments: +- AWS, Azure, and GCP auth engines are not bundled in OpenBao's core binary; they are separate plugins (`openbao/openbao-plugins`) that must be registered and mounted before those cloud login paths can be used. +- Kubernetes and AppRole auth engines are built-in to OpenBao out of the box. +- For AppRole authentication, Warpgate supports both static Secret IDs and Vault Response Wrapping tokens (specify `unwrap:` in the credential file). +- **Audit devices cannot be enabled over the API** — OpenBao answers `cannot + enable audit device via API; use declarative, config-based audit device + management instead`. Declare one in the server config, where `type` and `path` + are both required and the device's own settings belong under `options`; a + `file_path` written at the top level is accepted with only a warning and then + ignored, which looks exactly like a working audit device that never writes + anything. Worth knowing, because the issuance record on the Vault side is half + the point of this feature. + +## Operational notes + +**Vault is on the critical path of every session.** An outage locks every target +using certificate auth — including the one you would use to reach Vault. Keep a +break-glass path: one target on stored-key auth, or console access. + +**Clock skew.** Certificates live minutes. A target whose clock lags rejects them +as not yet valid, with an error that does not say so. Require NTP on targets. + +**Certificate TTL vs. session length.** The TTL bounds how long the certificate +may be *presented*, not how long the session lasts. An established session +survives expiry. + +**Revocation.** Stopping issuance at Vault takes effect for new sessions only. An +already-issued certificate stays valid for its TTL — which is why the TTL is +minutes. + +**Warpgate does not rate-limit issuance.** One session is one signing call, and +nothing on the Warpgate side caps how many a user can start. Put the ceiling in +Vault — a rate limit quota on the mount (`vault write sys/quotas/rate-limit/ssh +path=ssh-client-signer rate=...`) bounds it without depending on Warpgate +behaving. + +**A slow Vault delays every session, not just one.** Warpgate holds one client +token and logs in once for everybody; the login is serialised so an expiring +token cannot send every session in flight to Vault at once. The cost is that a +login which hangs blocks the others for up to `timeout`. Keep `timeout` at a +value you are willing to make every concurrent session wait. + +**Critical options are refused unless the target names them.** If a Vault role +sets `default_critical_options`, a `force-command` there replaces whatever the +user typed — the target runs Vault's command, under the user's own principal and +key ID, and the recording shows its output. Planting one needs write access to a +role, which is a lower bar than the right to sign with it or a route to the +target, so Warpgate is the only place this can be caught. + +A target therefore refuses any critical option it was not told to expect. Where a +role sets one deliberately, name it on the target — and pin the value, since for +`force-command` the value is the command: + +```json +"auth": { + "kind": "Certificate", + "role": "warpgate-backup", + "allowed_critical_options": [ + { "name": "force-command", "value": "/usr/local/bin/backup" } + ] +} +``` + +**Pinning a `value` also makes the option mandatory.** A certificate that omits +`force-command` above is refused, not accepted as unrestricted — otherwise the +role-write attack works just as well in reverse: remove the option instead of +adding one, and a target locked to a single command hands out a shell. + +Leaving `value` unset only permits: the option may appear with any value, and a +certificate without it is fine. That is how you express a role that *sometimes* +sets an option — `source-address`, say. If every entry were mandatory there +would be no way to say it: list the option and certificates without it fail, +omit it and certificates with it fail. + +The refusal reaches the connecting user, not only the log. + +**Extensions are not checked at all, and `force-command` alone does not confine a +session.** An unexpected extension is logged at debug level and nothing else. + +That is a gap, not a design decision, and it undercuts the example above. A +`force-command` decides what the *shell or exec* channel runs. It does not touch +the other channel types, and OpenSSH gates those purely on what the certificate +carries: `permit-port-forwarding` opens `direct-tcpip`, `permit-agent-forwarding` +reaches the connecting user's own SSH agent. So a role whose +`default_extensions` grants either — set deliberately, or written by someone with +role-write and nothing else — gives a session that forwards and pivots from a +target the certificate was supposed to lock to one command, and Warpgate accepts +it without comment. + +Until this is closed, the control is on the Vault side only: keep +`default_extensions` to the minimum the role actually needs, as §1 says, and do +not read a pinned `force-command` as confinement on its own. Check the target's +own `sshd_config` too — `AllowTcpForwarding no` and `AllowAgentForwarding no` +hold regardless of what a certificate permits. + +**`address` and `metadata_address` are fetched as given.** Both come from +`warpgate.yaml`, so anyone who can edit that file can point Warpgate's outbound +requests — including the ones carrying its Vault token and its cloud identity +token — at a host of their choosing. That is no more access than editing the +config already grants, but it means the config file deserves the same protection +as a credential: root-owned, not world-readable, and under change review. +Warpgate refuses plain HTTP for `address` outside localhost and never follows a +redirect, so neither address can be downgraded or re-pointed after the fact by +whatever answers it. diff --git a/warpgate-vault/proptest-regressions/client.txt b/warpgate-vault/proptest-regressions/client.txt new file mode 100644 index 000000000..a7dd6e298 --- /dev/null +++ b/warpgate-vault/proptest-regressions/client.txt @@ -0,0 +1,9 @@ +# Seeds for failure cases proptest has generated in the past. It is +# automatically read and these particular cases re-run before any +# novel cases are generated. +# +# It is recommended to check this file in to source control so that +# everyone who runs the test benefits from these saved cases. +cc 851e2880b0c22806f6c817fc15065aa1a3f426aaa1f64fa86333f607cc891964 # shrinks to scheme = "http", host = "127.0.0.2", port = None +cc 360faa84ff02441e45a56b85c41868f5bbdf7b3fda1fbbf0bcb51068e3e2e1a3 # shrinks to principal = "," +cc d31ee51a02c1d0381b002986722a67b1908d67d0b27cab404dcde0c41c638655 # shrinks to address = "A:" diff --git a/warpgate-vault/src/client.rs b/warpgate-vault/src/client.rs new file mode 100644 index 000000000..35b3b14da --- /dev/null +++ b/warpgate-vault/src/client.rs @@ -0,0 +1,1801 @@ +use std::path::Path; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::time::{Duration, Instant}; + +use data_encoding::BASE64; +use reqwest::StatusCode; +use serde::{Deserialize, Serialize}; +use tokio::sync::Mutex; +use tracing::{debug, warn}; +use warpgate_common::{MAX_CERTIFICATE_LIFETIME, VaultAuth, VaultConfig}; +use zeroize::{Zeroize, Zeroizing}; + +use crate::error::{Result, VaultError}; +use crate::metadata; + +/// Re-login this long before the lease actually runs out, so a certificate +/// request issued just after the check cannot race the expiry. +const TOKEN_EXPIRY_MARGIN: Duration = Duration::from_secs(30); + +/// How much of a failed response is kept for the error message. Anything +/// answering on the configured address can send an arbitrarily large body, so +/// this bounds what a bad endpoint can make Warpgate allocate. +const MAX_ERROR_BODY: usize = 256; + +/// The largest response body accepted from a successful call. A signed +/// certificate is a couple of kilobytes and an auth response smaller still, so +/// the bound is far above anything Vault sends — but without one, an endpoint +/// answering on the configured address can make every session in flight buffer +/// a body of its choosing. +const MAX_RESPONSE_BODY: usize = 256 * 1024; + +static NEXT_TOKEN_ID: AtomicU64 = AtomicU64::new(1); + +/// Whether this configuration leaves the AWS login request replayable, and why. +/// +/// `server_id` is bound into the SigV4 signature as +/// `X-Vault-AWS-IAM-Server-ID`, and Vault checks it against its own +/// `iam_server_id_header_value`. Without it the signed request proves only +/// "this principal", so anyone who obtains one can present it to any other +/// Vault that trusts the same principal — which is exactly what the README says +/// this field prevents, while the field defaulted to unset and said nothing. +/// +/// A warning rather than a refusal, matching what this constructor already does +/// for a plain-HTTP loopback address: Vault ignores the header entirely unless +/// `iam_server_id_header_value` is configured, so requiring a value here would +/// force operators to invent one for a server that will not look at it. +fn aws_binding_advice(auth: &VaultAuth) -> Option<&'static str> { + match auth { + VaultAuth::Aws { + server_id: None, .. + } => Some( + "vault.auth.server_id is unset, so the signed AWS login request is not bound to \ + this Vault and can be replayed against any other that trusts the same principal. \ + Set it here and as iam_server_id_header_value on the Vault AWS auth mount.", + ), + _ => None, + } +} + +fn validate_address(address: &str) -> Result<()> { + let parsed = url::Url::parse(address).map_err(|e| VaultError::InvalidAddress(e.to_string()))?; + if parsed.scheme() != "https" { + // Matched on the parsed host rather than on `host_str`, which renders an + // IPv6 address with its brackets — `[::1]` never equalled `::1`, so a + // loopback Vault over IPv6 was refused as though it were remote. + let is_loopback = match parsed.host() { + Some(url::Host::Domain(domain)) => domain == "localhost", + Some(url::Host::Ipv4(ip)) => ip.is_loopback(), + Some(url::Host::Ipv6(ip)) => ip.is_loopback(), + None => false, + }; + if !is_loopback { + return Err(VaultError::InsecureAddress); + } + } + Ok(()) +} + +fn validate_segment(name: &str) -> Result<()> { + // The rule itself lives in `warpgate-common`, so the admin API can refuse a + // name at save time by the same test the signing path applies at connect + // time. Two copies of a rule are two rules eventually. + if !warpgate_common::vault_name_is_well_formed(name) { + return Err(VaultError::InvalidRole(name.to_string())); + } + Ok(()) +} + +/// Vault reads `valid_principals` as a comma-separated list, so a comma in the +/// target's username would silently widen the certificate to accounts the +/// operator never named. +fn validate_principal(principal: &str) -> Result<()> { + if principal.is_empty() || principal.contains(',') || principal.chars().any(char::is_control) { + return Err(VaultError::InvalidPrincipal(principal.to_owned())); + } + Ok(()) +} + +/// Enough for `warpgate::` with a username nobody would call +/// unreasonable, and far below anything that would matter in a log line. +const MAX_KEY_ID: usize = 256; + +/// The key ID is echoed verbatim into the target's own sshd log — which is the +/// point, since that is what makes a proxied session attributable to a person. +/// +/// Both halves matter, and the second was missing. A control character would +/// let a Warpgate username forge log lines there; an unbounded length lets it +/// bury them. A 4 KB username produced a 4 KB key ID, which Vault signed and +/// the target wrote out on every connection — and the check on the *returned* +/// key ID could not see it, because it compares against what was asked for. +fn validate_key_id(key_id: &str) -> Result<()> { + if key_id.chars().any(char::is_control) || key_id.len() > MAX_KEY_ID { + return Err(VaultError::InvalidKeyId); + } + Ok(()) +} + +/// Keeps the error message bounded and always a valid string: `from_utf8_lossy` +/// replaces a multi-byte character cut in half by the byte limit instead of +/// panicking the way slicing a `String` at a non-boundary would. +fn render_error_body(bytes: &[u8], truncated: bool) -> String { + let mut text = String::from_utf8_lossy(bytes).into_owned(); + if truncated { + text.push_str("... (truncated)"); + } + text +} + +struct CachedToken { + id: u64, + value: Zeroizing, + /// `None` for a token Vault says has no lease. + /// + /// Nothing here revokes anything. What a `403` on signing does is drop this + /// entry so the next call authenticates again — cache invalidation, not + /// revocation, and the earlier wording said the second. Warpgate never + /// calls `auth/token/revoke-self`, and the reason it does not is that the + /// one moment it would want to is exactly when the token has just been + /// refused: a token Vault rejects for signing would as likely be rejected + /// for revoking, so the call would fail and the token is already useless. + /// Vault expires it at the end of its lease either way. + expires_at: Option, +} + +/// A secret ID that was delivered response-wrapped, kept alongside the exact +/// file content it came from. +/// +/// A wrapping token can be redeemed once, while the secret ID inside it stays +/// usable until its own `secret_id_num_uses` or `secret_id_ttl` runs out — so +/// unwrapping per login would fail every login after the first. Keying on the +/// file content means a freshly provisioned token is still picked up: the file +/// is read on every login, only the redemption is skipped. +struct UnwrappedSecretId { + source: Zeroizing, + secret_id: Zeroizing, +} + +/// Room reserved for a login payload before anything is written into it. A +/// service account token or a signed AWS header set is a few kilobytes; this is +/// far above that, and the reason it matters is below. +const LOGIN_PAYLOAD_CAPACITY: usize = 32 * 1024; + +/// The largest credential file that will be read. A service account token or a +/// wrapped secret ID is a few kilobytes; beyond this the file is not a +/// credential, and reading it would outgrow the payload buffer reserved above +/// and reintroduce the grow-and-copy leak that reservation exists to prevent. +const MAX_CREDENTIAL_FILE: u64 = 16 * 1024; + +/// Serializes a login payload into a buffer that is zeroized on drop. +/// +/// Public so that `tests/zeroization.rs` can exercise *this* function rather +/// than reimplementing the safe pattern beside it: a test that rebuilds the +/// pattern inline stays green when the real one is reverted, which is exactly +/// what it exists to prevent. +/// +/// Written into a buffer reserved up front rather than through +/// `serde_json::to_string`, because a `String` that grows while being written +/// frees each smaller buffer without wiping it — leaving a prefix of the +/// credential-bearing JSON in freed memory on every single login. `Zeroizing` +/// only ever wipes the buffer that survives to the end. Measured, with the +/// mechanism narrowed down, in `tests/zeroization.rs`. +pub fn login_payload(value: &T) -> Result>> { + let mut buffer = Zeroizing::new(Vec::with_capacity(LOGIN_PAYLOAD_CAPACITY)); + serde_json::to_writer(&mut *buffer, value)?; + Ok(buffer) +} + +/// Wipes the signed AWS headers on every path out of the function that holds +/// them. +/// +/// They carry the SigV4 signature and, on an instance role, the session token. +/// The wipe used to be a loop at the end, after a fallible serialization — so +/// the one path where something had already gone wrong was the path that +/// skipped it. Ordering is not a property worth relying on for this; `Drop` is. +/// +/// Not covered by a test, deliberately rather than by omission. Once `Drop` has +/// run the values are gone, so there is nothing left to assert against; a test +/// that wipes a map itself and then reads it proves only that `zeroize` works, +/// which is the shape this crate has already been caught writing once. The +/// change here is that the wipe cannot be skipped, and that is a property of +/// `Drop` rather than of anything a test can observe. +struct WipedHeaders(warpgate_aws::StsIdentityRequest); + +impl Drop for WipedHeaders { + fn drop(&mut self) { + for value in self.0.headers.values_mut() { + value.zeroize(); + } + } +} + +#[derive(Serialize)] +struct JwtLogin<'a> { + role: &'a str, + jwt: &'a str, +} + +#[derive(Serialize)] +struct AppRoleLogin<'a> { + role_id: &'a str, + secret_id: &'a str, +} + +#[derive(Serialize)] +struct AzureLogin<'a> { + role: &'a str, + jwt: &'a str, + subscription_id: &'a str, + resource_group_name: &'a str, + vm_name: &'a str, + #[serde(skip_serializing_if = "Option::is_none")] + vmss_name: Option<&'a str>, +} + +#[derive(Serialize)] +struct AwsLogin<'a> { + role: Option<&'a str>, + iam_http_request_method: &'a str, + iam_request_url: &'a str, + iam_request_body: &'a str, + iam_request_headers: &'a str, +} + +#[derive(Serialize)] +struct SignRequest<'a> { + public_key: &'a str, + valid_principals: &'a str, + cert_type: &'a str, + key_id: &'a str, + /// Omitted rather than sent as zero when unset: Vault reads a zero TTL as + /// "use the role's default", but only if the field is absent. + #[serde(skip_serializing_if = "Option::is_none")] + ttl: Option, +} + +#[derive(Deserialize)] +struct AuthResponse { + auth: AuthData, +} + +#[derive(Deserialize)] +struct AuthData { + client_token: String, + lease_duration: u64, +} + +#[derive(Deserialize)] +struct SignResponse { + data: SignData, +} + +#[derive(Deserialize)] +struct SignData { + signed_key: String, +} + +#[derive(Deserialize)] +struct UnwrapResponse { + data: Option, +} + +#[derive(Deserialize)] +struct UnwrapData { + secret_id: Option, +} + +/// Reads a response into a bounded, zeroized buffer. +/// +/// At module level and `pub(crate)` because the metadata calls answer to the +/// same argument as the Vault ones: an endpoint named in the configuration is +/// not a trusted party, and what it returns is a credential. This bound was +/// added for the Vault client and not mirrored there, which is the kind of +/// half-applied fix this file has produced before. +/// Makes room for `incoming` more bytes without leaving the old buffer behind. +/// +/// `Vec` grows by allocating, copying and freeing, and it frees the old block +/// itself — `Zeroizing` only ever wipes the one that survives to be dropped. So +/// a buffer reserved at 32 KiB and grown past it leaves a credential-bearing +/// copy in freed memory, which reserving up front was supposed to prevent and +/// only made less likely. +/// +/// That was raised in review, disputed on the grounds that the reservation was +/// a deliberate trade, and upheld on the same grounds. Neither side measured it. +/// `zeroization.rs` does, and finds the canary in a freed block. +/// +/// Moving to a new buffer by hand keeps the old one ours until it is dropped, +/// so its own `Zeroizing` wipes it before the allocator takes it back. +/// +/// Public so the test exercises this function rather than a copy of its shape +/// written beside it — the same reason `login_payload` is public, and the +/// mistake the first version of that test made. +#[must_use] +pub fn grown_without_leaving_a_copy( + buf: Zeroizing>, + incoming: usize, +) -> Zeroizing> { + if buf.len() + incoming <= buf.capacity() { + return buf; + } + let wanted = (buf.capacity() * 2).max(buf.len() + incoming); + let mut grown: Zeroizing> = Zeroizing::new(Vec::with_capacity(wanted)); + grown.extend_from_slice(&buf); + grown +} + +pub(crate) async fn read_bounded(mut response: reqwest::Response) -> Result>> { + // Reserved up front, for the reason spelled out on `login_payload`: a buffer + // that grows frees every size it outgrew without wiping it, and `Zeroizing` + // only clears the one that survives. Every caller of this function carries a + // credential — the Vault token, an AppRole secret ID, a cloud identity JWT — + // so the response path needed the same treatment as the request path and did + // not get it. A GCE `format=full` identity token is around 2 KiB and + // arrives in more than one chunk, which is exactly the case that reallocs. + // + // Not `MAX_RESPONSE_BODY`: that is the refusal threshold, not an + // expectation. Anything past this reserve is far larger than a credential. + let mut buf: Zeroizing> = Zeroizing::new(Vec::with_capacity(LOGIN_PAYLOAD_CAPACITY)); + while let Some(chunk) = response.chunk().await? { + if buf.len() + chunk.len() > MAX_RESPONSE_BODY { + return Err(VaultError::OversizedResponse); + } + // Grown by hand, because `Vec` frees the old allocation itself and + // `Zeroizing` only ever sees the one that survives. Reserving up front + // was supposed to make growth impossible; it only made it unlikely, and + // a response between the reserve and the refusal threshold reallocated + // with a credential in it. Measured, after being argued about twice and + // dismissed both times: `zeroization.rs` finds the canary in a freed + // block for a 64 KiB body. + // + // Moving to a new buffer explicitly means the old one is dropped while + // it is still ours, so its own `Zeroizing` wipes it before the allocator + // gets it back. + buf = grown_without_leaving_a_copy(buf, chunk.len()); + buf.extend_from_slice(&chunk); + } + Ok(buf) +} + +pub(crate) async fn read_bounded_json( + response: reqwest::Response, +) -> Result { + let buf = read_bounded(response).await?; + Ok(serde_json::from_slice(&buf)?) +} + +pub struct VaultClient { + config: VaultConfig, + http: reqwest::Client, + metadata_http: reqwest::Client, + token: Mutex>, + unwrapped_secret_id: Mutex>, +} + +impl VaultClient { + pub fn new(config: VaultConfig) -> Result { + validate_address(&config.address)?; + validate_segment(&config.mount)?; + validate_segment(&config.default_role)?; + + // Both ends of the range, caught here rather than at connect time. + // + // A sub-second TTL truncates to "0s", which both Vault and OpenBao + // refuse. One above the ceiling is refused on arrival by the certificate + // check instead. Either way the mistake would otherwise surface as a + // failed session for every target at once, with nothing pointing at the + // config line that caused it — and that argument, written here for the + // lower bound, applies identically to the upper one, which is the half + // it was not applied to. + if let Some(ttl) = config.certificate_ttl + && (ttl.as_secs() == 0 || ttl > MAX_CERTIFICATE_LIFETIME) + { + return Err(VaultError::InvalidCertificateTtl(ttl)); + } + + if let Some(advice) = aws_binding_advice(&config.auth) { + warn!("{advice}"); + } + + if !config.address.starts_with("https://") { + // Only reachable for a loopback address, which validation allows so + // a development Vault does not need a certificate. Said out loud + // because the token crosses this connection in a header. + warn!( + address = config.address, + "The Vault address is not HTTPS; the client token is sent in clear text" + ); + } + + // Redirects are refused rather than followed: reqwest strips + // `Authorization` on a cross-origin hop but knows nothing about + // `X-Vault-Token`, so a 307 from a hostile or misconfigured endpoint + // would replay the token to another host, or downgrade the request to + // plain HTTP. + // A Vault behind a private CA. Added to the host's trust store rather + // than replacing it, and read here so an unreadable or malformed bundle + // is a startup error naming the file, not a signing failure on every + // target at once with nothing pointing at the config line. + let mut builder = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .timeout(config.timeout); + if let Some(path) = config.ca_bundle.as_ref() { + let pem = std::fs::read(path).map_err(|source| VaultError::CaBundle { + path: path.clone(), + reason: source.to_string(), + })?; + for certificate in reqwest::Certificate::from_pem_bundle(&pem).map_err(|source| { + VaultError::CaBundle { + path: path.clone(), + reason: source.to_string(), + } + })? { + builder = builder.add_root_certificate(certificate); + } + } + let http = builder.build()?; + + // The metadata services are link-local and plain HTTP by definition, and + // reqwest honours HTTP_PROXY/HTTPS_PROXY by default — a proxy in the + // environment would carry the instance identity token off the host. + // GCE's default address is a hostname, so the usual IP-based NO_PROXY + // list does not cover it. Vault's own address keeps ambient proxy + // support, since reaching it through one is a legitimate deployment. + let metadata_http = reqwest::Client::builder() + .redirect(reqwest::redirect::Policy::none()) + .no_proxy() + .timeout(config.timeout) + .build()?; + + Ok(Self { + config, + http, + metadata_http, + token: Mutex::new(None), + unwrapped_secret_id: Mutex::new(None), + }) + } + + pub fn default_role(&self) -> &str { + &self.config.default_role + } + + /// How long any single call to Vault may take. + /// + /// Exposed because the connection path has to budget for several of them. + pub const fn timeout(&self) -> Duration { + self.config.timeout + } + + /// The lifetime asked for when signing, if the operator set one. + /// + /// Exposed because asking is not getting: what comes back is checked + /// against it. + pub const fn certificate_ttl(&self) -> Option { + self.config.certificate_ttl + } + + /// The signing CA the operator pinned, if any. + /// + /// Exposed for the same reason as the TTL, one step further: the crate + /// speaks Vault and returns an OpenSSH string, so the party that parses + /// certificates is the one that compares them. + pub fn pinned_ca_public_key(&self) -> Option<&str> { + self.config.ca_public_key.as_deref() + } + + /// Signs `public_key` into a short-lived OpenSSH user certificate, returned + /// in OpenSSH wire format. + /// + /// `principals` are usernames on the target, not Warpgate usernames. `key_id` + /// is echoed into the target's own sshd log, which is what makes a session + /// attributable to a person rather than to the gateway. + pub async fn sign_ssh_key( + &self, + role: &str, + public_key: &str, + principals: &str, + key_id: &str, + ) -> Result { + validate_segment(role)?; + validate_principal(principals)?; + validate_key_id(key_id)?; + + let (token_id, result) = self.sign_once(role, public_key, principals, key_id).await; + + match result { + // A cached token can stop being accepted long before its lease runs + // out — it may have been revoked, or Vault resealed or restarted. + // Forcing one re-login tells that apart from a real policy denial. + // Only invalidate if the token in cache is still the token that failed. + Err(VaultError::Api { status, .. }) if status == StatusCode::FORBIDDEN => { + let mut guard = self.token.lock().await; + if guard.as_ref().map(|t| t.id) == Some(token_id) { + *guard = None; + } + drop(guard); + let (_, second_try) = self.sign_once(role, public_key, principals, key_id).await; + second_try + } + res => res, + } + } + + async fn sign_once( + &self, + role: &str, + public_key: &str, + principals: &str, + key_id: &str, + ) -> (u64, Result) { + let (token_id, token) = match self.token().await { + Ok(t) => t, + Err(err) => return (0, Err(err)), + }; + + let response = self + .http + .post(self.url(&format!("{}/sign/{role}", self.config.mount))) + .header("X-Vault-Token", token.as_str()) + .json(&SignRequest { + public_key, + valid_principals: principals, + cert_type: "user", + key_id, + ttl: self + .config + .certificate_ttl + .map(|ttl| format!("{}s", ttl.as_secs())), + }) + .send() + .await; + + let res = match response { + Ok(resp) => match Self::check(resp).await { + Ok(resp) => match Self::read_json::(resp).await { + Ok(parsed) => { + debug!(role, principals, key_id, "Issued an SSH certificate"); + Ok(parsed.data.signed_key) + } + Err(e) => Err(e), + }, + Err(e) => Err(e), + }, + Err(e) => Err(VaultError::Request(e)), + }; + + (token_id, res) + } + + /// The cached client token, logging in again once it nears expiry. Plain + /// re-login is used instead of Vault's renew API because it behaves the same + /// whether or not the token has hit its `max_ttl`. + /// + /// The lock is deliberately held across the login: one expiring token would + /// otherwise send every session in flight to Vault at once, and for the + /// cloud methods each of those first fetches an identity token of its own. + /// The cost of that choice is that a slow login delays every other session + /// by up to `timeout`. + async fn token(&self) -> Result<(u64, Zeroizing)> { + let mut cached = self.token.lock().await; + + if let Some(token) = cached.as_ref() + && token.expires_at.is_none_or(|at| Instant::now() < at) + { + return Ok((token.id, token.value.clone())); + } + + // Bounded here rather than inside `login()`, because the bound has to + // cover assembling the request as well as sending it. `reqwest` times + // out the POST; `login_body()` reads a credential — a file for the + // Kubernetes and AppRole methods, the whole SDK credential chain for + // AWS — and nothing bounded that. Since this lock is held across the + // login on purpose, an unbounded read here is not one stalled session + // but all of them. + let token = tokio::time::timeout(self.config.timeout, self.login()) + .await + .map_err(|_| VaultError::LoginTimeout)??; + let id = token.id; + let value = token.value.clone(); + *cached = Some(token); + Ok((id, value)) + } + + async fn login(&self) -> Result { + let (path, body) = self.login_body().await?; + + let response = self + .http + .post(self.url(path)) + .header(reqwest::header::CONTENT_TYPE, "application/json") + .body(body.to_vec()) + .send() + .await?; + let auth = Self::read_json::(Self::check(response).await?) + .await? + .auth; + + debug!(method = self.config.auth.kind(), "Authenticated to Vault"); + + let token_id = NEXT_TOKEN_ID.fetch_add(1, Ordering::Relaxed); + Ok(CachedToken { + id: token_id, + value: Zeroizing::new(auth.client_token), + // Vault reports a lease of zero for a token that does not expire. + // Reading that as "expired half a minute ago" would turn every + // certificate request into a fresh login. + // + // The lease is an unbounded number out of a network response, and + // `Instant + Duration` panics on overflow, so an absurd one is + // rejected as a bad response rather than taken at its word. + expires_at: match auth.lease_duration { + 0 => None, + seconds => Some( + Instant::now() + .checked_add( + Duration::from_secs(seconds).saturating_sub(TOKEN_EXPIRY_MARGIN), + ) + .ok_or(VaultError::InvalidLease(seconds))?, + ), + }, + }) + } + + /// The endpoint and JSON payload for the configured authentication method. + /// + /// The payload is built through `serde` straight into a `Zeroizing` buffer + /// rather than through a `serde_json::Value`: every intermediate copy of a + /// service account token, secret ID or identity token is one more place the + /// credential outlives the request in freed memory. The buffer reqwest takes + /// to put the body on the wire is the one copy Warpgate cannot reach. + async fn login_body(&self) -> Result<(&'static str, Zeroizing>)> { + Ok(match &self.config.auth { + VaultAuth::Kubernetes { role, token_path } => { + let jwt = Self::read_credential(token_path).await?; + ( + "auth/kubernetes/login", + login_payload(&JwtLogin { role, jwt: &jwt })?, + ) + } + VaultAuth::AppRole { + role_id, + secret_id_path, + } => { + let cred = Self::read_credential(secret_id_path).await?; + let secret_id = if cred.starts_with("unwrap:") { + self.unwrapped_secret_id(secret_id_path, &cred).await? + } else { + cred + }; + ( + "auth/approle/login", + login_payload(&AppRoleLogin { + role_id: role_id.expose_secret(), + secret_id: &secret_id, + })?, + ) + } + VaultAuth::Aws { + role, + server_id, + region, + } => ( + "auth/aws/login", + self.aws_login_body(role.as_deref(), server_id.as_deref(), region.as_deref()) + .await?, + ), + VaultAuth::Azure { + role, + resource, + metadata_address, + } => { + let (jwt, instance) = + metadata::azure_login_material(&self.metadata_http, metadata_address, resource) + .await?; + ( + "auth/azure/login", + login_payload(&AzureLogin { + role, + jwt: &jwt, + subscription_id: &instance.subscription_id, + resource_group_name: &instance.resource_group_name, + vm_name: &instance.name, + vmss_name: (!instance.vm_scale_set_name.is_empty()) + .then_some(instance.vm_scale_set_name.as_str()), + })?, + ) + } + VaultAuth::Gcp { + role, + metadata_address, + } => { + let audience = format!("vault/{role}"); + let jwt = + metadata::gcp_identity_token(&self.metadata_http, metadata_address, &audience) + .await?; + ( + "auth/gcp/login", + login_payload(&JwtLogin { role, jwt: &jwt })?, + ) + } + }) + } + + /// The secret ID behind a `unwrap:` file, redeeming the wrapping + /// token the first time and on every later change to the file. + async fn unwrapped_secret_id( + &self, + path: &Path, + cred: &Zeroizing, + ) -> Result> { + let mut cached = self.unwrapped_secret_id.lock().await; + + if let Some(entry) = cached.as_ref() + && entry.source.as_str() == cred.as_str() + { + return Ok(entry.secret_id.clone()); + } + + let wrapping_token = Zeroizing::new(cred.trim_start_matches("unwrap:").trim().to_owned()); + let secret_id = self.unwrap_secret_id(&wrapping_token).await.map_err(|e| { + VaultError::SecretIdUnwrap { + path: path.to_owned(), + source: Box::new(e), + } + })?; + + *cached = Some(UnwrappedSecretId { + source: cred.clone(), + secret_id: secret_id.clone(), + }); + Ok(secret_id) + } + + async fn unwrap_secret_id(&self, wrapping_token: &str) -> Result> { + let response = self + .http + .post(self.url("sys/wrapping/unwrap")) + .header("X-Vault-Token", wrapping_token) + .send() + .await?; + + let unwrap_resp = Self::read_json::(Self::check(response).await?).await?; + + // Moved rather than copied out of the response, so the only allocation + // holding the secret ID is the one that gets zeroized on drop. + unwrap_resp + .data + .and_then(|data| data.secret_id) + .map_or_else( + || { + Err(VaultError::Api { + status: StatusCode::BAD_REQUEST, + body: "failed to unwrap secret_id from Vault response".to_owned(), + }) + }, + |secret_id| Ok(Zeroizing::new(secret_id)), + ) + } + + async fn aws_login_body( + &self, + role: Option<&str>, + server_id: Option<&str>, + region: Option<&str>, + ) -> Result>> { + let request = + WipedHeaders(warpgate_aws::sign_sts_identity_request(region, server_id).await?); + + // The headers carry the SigV4 signature and, on an instance role, the + // session token — the one part of this request worth protecting. The URL + // and body are the same public constants on every call. Every buffer + // they pass through is zeroized, to match what the other methods do with + // their credentials. + // Through the same sized buffer `login_payload` uses, and for the same + // reason: `serde_json::to_string` grows its `String` as it writes and + // frees each smaller one unwiped. These headers carry the SigV4 + // signature and, on an instance role, the session token — so this was + // the one call site still doing what the comment on `login_payload` + // forbids, one function away from it. + let mut headers = Zeroizing::new(Vec::with_capacity(LOGIN_PAYLOAD_CAPACITY)); + serde_json::to_writer(&mut *headers, &request.0.headers)?; + let encoded_headers = Zeroizing::new(BASE64.encode(&headers)); + + login_payload(&AwsLogin { + role, + iam_http_request_method: request.0.method, + iam_request_url: &BASE64.encode(request.0.url.as_bytes()), + iam_request_body: &BASE64.encode(request.0.body.as_bytes()), + iam_request_headers: &encoded_headers, + }) + } + + /// `read_to_string` sizes its buffer from the file's own length, so it does + /// not grow while reading and leaves nothing behind — measured, rather than + /// assumed, in `tests/zeroization.rs`. That holds only while the file is + /// small enough to fit the reserved payload buffer, hence the cap. + async fn read_credential(path: &Path) -> Result> { + let describe = |source| VaultError::CredentialFile { + path: path.to_owned(), + source, + }; + + // One handle, opened once, and a bound on the stream rather than on + // what a separate `stat` claimed. + // + // It used to call `metadata(path)` and then `read_to_string(path)`, + // which opens the path a second time: the file can be replaced between + // the two, and a FIFO reports a length of zero and then delivers as much + // as it likes — while the token mutex is held across login, so blocking + // here stalls every session at once rather than one. + let file = tokio::fs::File::open(path).await.map_err(describe)?; + let size = file.metadata().await.map_err(describe)?.len(); + if size > MAX_CREDENTIAL_FILE { + return Err(VaultError::CredentialTooLarge { + path: path.to_owned(), + size, + }); + } + + // Reserved at the cap rather than at the reported size: a buffer that + // grows frees every smaller size unwiped, and the reported size is the + // number this function no longer trusts. + let mut raw = Zeroizing::new(String::with_capacity(MAX_CREDENTIAL_FILE as usize + 1)); + use tokio::io::AsyncReadExt as _; + let read = file + .take(MAX_CREDENTIAL_FILE + 1) + .read_to_string(&mut raw) + .await + .map_err(describe)? as u64; + if read > MAX_CREDENTIAL_FILE { + return Err(VaultError::CredentialTooLarge { + path: path.to_owned(), + size: read, + }); + } + Ok(Zeroizing::new(raw.trim().to_owned())) + } + + fn url(&self, path: &str) -> String { + format!("{}/v1/{path}", self.config.address.trim_end_matches('/')) + } + + /// Parses a successful response, refusing one larger than any Vault answer + /// could be. `json()` would buffer whatever arrives before anything got to + /// look at its size, so a single endpoint could hold as much of Warpgate's + /// memory as it cared to send, once per session in flight. + async fn read_json(response: reqwest::Response) -> Result { + read_bounded_json(response).await + } + + async fn check(response: reqwest::Response) -> Result { + let status = response.status(); + if status.is_success() { + return Ok(response); + } + Err(VaultError::Api { + status, + body: Self::error_body(response).await, + }) + } + + /// The first `MAX_ERROR_BODY` bytes of a failed response. Read chunk by + /// chunk rather than with `text()`, which would buffer the whole body before + /// there was anything to truncate — an endpoint answering on the configured + /// address could then make an error message cost arbitrary memory. + async fn error_body(mut response: reqwest::Response) -> String { + // Wiped and reserved up front, like the credential path beside it. + // Vault's error bodies are its own words rather than a secret, but they + // are read off the same connection as the token and this is the last + // buffer on that path without the treatment. The reserve is what makes + // the wipe complete: a `Vec` that grows frees every size it outgrew + // without wiping it, and `Zeroizing` only ever sees the one that lives. + let mut buf: Zeroizing> = Zeroizing::new(Vec::with_capacity(MAX_ERROR_BODY)); + let mut truncated = false; + + while buf.len() < MAX_ERROR_BODY { + match response.chunk().await { + Ok(Some(chunk)) => { + let room = MAX_ERROR_BODY.saturating_sub(buf.len()); + let taken = chunk.get(..room).unwrap_or(&chunk); + truncated |= taken.len() < chunk.len(); + buf.extend_from_slice(taken); + } + Ok(None) | Err(_) => break, + } + } + + // The loop can fill the buffer exactly while the body continues; one + // more poll is what tells a 256-byte body from a truncated one. + if !truncated && buf.len() == MAX_ERROR_BODY { + truncated = matches!(response.chunk().await, Ok(Some(chunk)) if !chunk.is_empty()); + } + + render_error_body(&buf, truncated) + } +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + use std::sync::{Arc, Mutex as StdMutex}; + + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::TcpListener; + + use super::*; + + /// A directory that goes away when the test does. + /// + /// These paths used to be `wg-