Merge pull request #34072 from alex-hunt-materialize/version_gate_readyz_probe

Version-gate the HTTP /api/readyz probe

Author: alex-hunt-materialize

bin/ci-builder

@@ -259,6 +259,7 @@ case "$cmd" in
             --env PYPI_TOKEN
             --env RUST_MIN_STACK
             --env MZ_DEV_BUILD_SHA
+            --env MZ_GHCR
             # For Miri with nightly Rust
             --env ZOOKEEPER_ADDR
             --env KAFKA_ADDRS

ci/nightly/pipeline.template.yml

@@ -2401,9 +2401,11 @@ steps:
               composition: orchestratord
               args: [--action=upgrade, --properties=individual, --runtime=3600, --recreate-cluster]
               ci-builder: stable
+        env:
+          # Old versions are not on GHCR yet
+          MZ_GHCR: 0
         agents:
           queue: hetzner-aarch64-8cpu-16gb
-        skip: "https://github.com/MaterializeInc/database-issues/issues/9891"
 
       - id: orchestratord-upgrade-combine
         label: "Orchestratord test (upgrade, combine props)"
@@ -2414,9 +2416,11 @@ steps:
               composition: orchestratord
               args: [--action=upgrade, --properties=combine, --runtime=3600, --recreate-cluster]
               ci-builder: stable
+        env:
+          # Old versions are not on GHCR yet
+          MZ_GHCR: 0
         agents:
           queue: hetzner-aarch64-8cpu-16gb
-        skip: "https://github.com/MaterializeInc/database-issues/issues/9891"
 
       - id: orchestratord-upgrade-chain-individual
         label: "Orchestratord test (upgrade chain, individual props)"
@@ -2427,9 +2431,11 @@ steps:
               composition: orchestratord
               args: [--action=upgrade-chain, --properties=individual, --runtime=3600, --recreate-cluster]
               ci-builder: stable
+        env:
+          # Old versions are not on GHCR yet
+          MZ_GHCR: 0
         agents:
           queue: hetzner-aarch64-8cpu-16gb
-        skip: "https://github.com/MaterializeInc/database-issues/issues/9891"
 
       - id: orchestratord-upgrade-chain-combine
         label: "Orchestratord test (upgrade chain, combine props)"
@@ -2440,6 +2446,8 @@ steps:
               composition: orchestratord
               args: [--action=upgrade-chain, --properties=combine, --runtime=3600, --recreate-cluster]
               ci-builder: stable
+        env:
+          # Old versions are not on GHCR yet
+          MZ_GHCR: 0
         agents:
           queue: hetzner-aarch64-16cpu-32gb
-        skip: "https://github.com/MaterializeInc/database-issues/issues/9891"

src/orchestratord/src/controller/materialize/environmentd.rs

@@ -21,7 +21,8 @@ use k8s_openapi::{
             Capabilities, ConfigMap, ConfigMapVolumeSource, Container, ContainerPort, EnvVar,
             EnvVarSource, HTTPGetAction, KeyToPath, PodSecurityContext, PodSpec, PodTemplateSpec,
             Probe, SeccompProfile, Secret, SecretKeySelector, SecretVolumeSource, SecurityContext,
-            Service, ServiceAccount, ServicePort, ServiceSpec, Toleration, Volume, VolumeMount,
+            Service, ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, Toleration, Volume,
+            VolumeMount,
         },
         networking::v1::{
             IPBlock, NetworkPolicy, NetworkPolicyEgressRule, NetworkPolicyIngressRule,
@@ -1312,17 +1313,31 @@ fn create_environmentd_statefulset_object(
         args.extend(extra_args.iter().cloned());
     }
 
-    let probe = Probe {
-        initial_delay_seconds: Some(1),
-        failure_threshold: Some(12),
-        http_get: Some(HTTPGetAction {
-            path: Some("/api/readyz".to_string()),
-            port: IntOrString::Int(config.environmentd_internal_http_port.into()),
-            host: None,
-            scheme: None,
-            http_headers: None,
-        }),
-        ..Default::default()
+    let probe = if mz.meets_minimum_version(&V147_DEV0) {
+        Probe {
+            initial_delay_seconds: Some(1),
+            failure_threshold: Some(12),
+            http_get: Some(HTTPGetAction {
+                path: Some("/api/readyz".to_string()),
+                port: IntOrString::Int(config.environmentd_internal_http_port.into()),
+                host: None,
+                scheme: None,
+                http_headers: None,
+            }),
+            ..Default::default()
+        }
+    } else {
+        // Older versions, despite having the /api/readyz endpoint,
+        // would sometimes never return an HTTP 200 response.
+        Probe {
+            initial_delay_seconds: Some(1),
+            failure_threshold: Some(12),
+            tcp_socket: Some(TCPSocketAction {
+                host: None,
+                port: IntOrString::Int(config.environmentd_sql_port.into()),
+            }),
+            ..Default::default()
+        }
     };
 
     let security_context = if config.enable_security_context {

test/orchestratord/mzcompose.py

@@ -30,7 +30,7 @@
 import yaml
 from semver.version import Version
 
-from materialize import MZ_ROOT, ci_util, git, spawn
+from materialize import MZ_ROOT, ci_util, git, spawn, ui
 from materialize.mz_version import MzVersion
 from materialize.mzcompose.composition import (
     Composition,
@@ -1517,6 +1517,9 @@ def get_mods() -> Iterator[list[Modification]]:
             for mods in mods_it:
                 run_scenario([mods], definition)
         elif action == Action.Upgrade:
+            assert not ui.env_is_truthy(
+                "MZ_GHCR", "1"
+            ), "Manually set MZ_GHCR=0 as an environment variable for upgrade testing"
             assert args.runtime
             end_time = (
                 datetime.datetime.now() + datetime.timedelta(seconds=args.runtime)
@@ -1535,6 +1538,9 @@ def get_mods() -> Iterator[list[Modification]]:
                 ]
                 run_scenario(scenario, definition)
         elif action == Action.UpgradeChain:
+            assert not ui.env_is_truthy(
+                "MZ_GHCR", "1"
+            ), "Manually set MZ_GHCR=0 as an environment variable for upgrade testing"
             assert args.runtime
             end_time = (
                 datetime.datetime.now() + datetime.timedelta(seconds=args.runtime)