feat(system-tests): support API boundary node playnets on the local backend

rs/config/src/http_handler.rs

@@ -66,6 +66,16 @@ pub struct Config {
 
     /// Serving at most `max_tracing_flamegraph_concurrent_requests` requests concurrently for all endpoints under `/_/tracing/flamegraph`.
     pub max_tracing_flamegraph_concurrent_requests: usize,
+
+    /// PEM-encoded certificates to trust, in addition to the public roots
+    /// compiled into the replica, when connecting to an API boundary node.
+    ///
+    /// A cloud engine subnet fetches its NNS delegation from an API boundary
+    /// node over TLS. In a testnet that node's certificate comes from a
+    /// throw-away CA rather than a public one, so that CA has to be handed to
+    /// the replica. Left unset in production, where the public roots are the
+    /// only trust anchors. To be used in system tests only.
+    pub extra_api_boundary_node_trust_anchors_pem: Option<String>,
 }
 
 impl Default for Config {
@@ -87,6 +97,7 @@ impl Default for Config {
             max_pprof_concurrent_requests: 5,
             ingress_message_certificate_timeout_seconds: 10,
             max_tracing_flamegraph_concurrent_requests: 5,
+            extra_api_boundary_node_trust_anchors_pem: None,
         }
     }
 }

rs/http_endpoints/nns_delegation_manager/src/nns_delegation_manager.rs

@@ -38,7 +38,10 @@ use ic_types::{
     time::expiry_time_from_now,
 };
 use rand::{Rng, seq::SliceRandom};
-use rustls::{ClientConfig, pki_types::ServerName};
+use rustls::{
+    ClientConfig,
+    pki_types::{CertificateDer, ServerName, pem::PemObject},
+};
 use tokio::{
     net::TcpStream,
     select,
@@ -389,6 +392,7 @@ async fn try_fetch_delegation_from_nns(
         CONNECTION_TIMEOUT,
         connect(
             log.clone(),
+            config,
             rt_handle,
             subnet_type,
             nns_subnet_id,
@@ -537,8 +541,36 @@ fn observe_delegation_sizes(builder: &NNSDelegationBuilder, metrics: &Delegation
         .observe(builder.flat_certificate_size_bytes() as f64);
 }
 
+/// The trust anchors used to authenticate an API boundary node: the public roots
+/// compiled into the replica, plus anything in `extra_anchors_pem`.
+///
+/// `extra_anchors_pem` is unset in production. It exists for testnets, where the
+/// API boundary node's certificate is issued by a throw-away CA rather than by a
+/// public one; see [`Config::extra_api_boundary_node_trust_anchors_pem`].
+fn api_boundary_node_root_store(
+    extra_anchors_pem: Option<&str>,
+) -> Result<rustls::RootCertStore, BoxError> {
+    let mut root_store =
+        rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
+
+    let Some(extra_anchors_pem) = extra_anchors_pem else {
+        return Ok(root_store);
+    };
+
+    for certificate in CertificateDer::pem_slice_iter(extra_anchors_pem.as_bytes()) {
+        let certificate = certificate
+            .map_err(|err| format!("Could not parse an extra API BN trust anchor: {err}"))?;
+        root_store
+            .add(certificate)
+            .map_err(|err| format!("Could not add an extra API BN trust anchor: {err}"))?;
+    }
+
+    Ok(root_store)
+}
+
 async fn connect(
     log: ReplicaLogger,
+    config: &Config,
     rt_handle: &tokio::runtime::Handle,
     subnet_type: SubnetType,
     nns_subnet_id: SubnetId,
@@ -604,8 +636,9 @@ async fn connect(
 
             let addr = SocketAddr::new(ip_addr, 443);
 
-            let root_store =
-                rustls::RootCertStore::from_iter(webpki_roots::TLS_SERVER_ROOTS.iter().cloned());
+            let root_store = api_boundary_node_root_store(
+                config.extra_api_boundary_node_trust_anchors_pem.as_deref(),
+            )?;
             let tls_client_config = rustls::ClientConfig::builder()
                 .with_root_certificates(root_store)
                 .with_no_client_auth();

rs/ic_os/config/tool/src/guestos/generate_ic_config.rs

@@ -29,6 +29,8 @@ pub struct IcConfigTemplate {
     pub domain_name: String,
     pub node_reward_type: String,
     pub malicious_behavior: String,
+    /// Already JSON-encoded: either `null` or a quoted string.
+    pub extra_api_boundary_node_trust_anchors_pem: String,
 }
 
 /// Generate IC configuration from template and guestos config
@@ -186,6 +188,18 @@ fn get_config_vars(guestos_config: &GuestOSConfig) -> Result<IcConfigTemplate> {
         .map(|mb| serde_json::to_string(mb).unwrap_or_default())
         .unwrap_or_default();
 
+    let extra_api_boundary_node_trust_anchors_pem = match &guestos_config
+        .guestos_settings
+        .guestos_dev_settings
+        .extra_api_boundary_node_trust_anchors_pem
+    {
+        // A PEM spans several lines, so it has to be JSON-encoded rather than
+        // interpolated verbatim.
+        Some(pem) => serde_json::to_string(pem)
+            .context("Failed to encode the extra API boundary node trust anchors")?,
+        None => "null".to_string(),
+    };
+
     Ok(IcConfigTemplate {
         // TODO https://dfinity.atlassian.net/browse/NODE-1909
         ipv6_prefix,
@@ -199,6 +213,7 @@ fn get_config_vars(guestos_config: &GuestOSConfig) -> Result<IcConfigTemplate> {
         domain_name,
         node_reward_type,
         malicious_behavior: with_default(malicious_behavior, "null"),
+        extra_api_boundary_node_trust_anchors_pem,
     })
 }
 

rs/ic_os/config/tool/templates/ic.json5.template

@@ -119,6 +119,7 @@
     // ====================================
     http_handler: {
         listen_addr: "[::]:8080",
+        extra_api_boundary_node_trust_anchors_pem: {{ extra_api_boundary_node_trust_anchors_pem }},
     },
 
     // ====================================

rs/ic_os/config/types/compatibility_tests/fixtures/guestos_v1.16.0.json

@@ -0,0 +1,55 @@
+{
+  "config_version": "1.16.0",
+  "network_settings": {
+    "ipv6_config": {
+      "Fixed": {
+        "address": "2a00:fb01:400:200::1/64",
+        "gateway": "2a00:fb01:400:200::1"
+      }
+    },
+    "ipv4_config": {
+      "address": "192.168.1.1",
+      "gateway": "192.168.1.254",
+      "prefix_length": 24
+    },
+    "domain_name": "ic.test"
+  },
+  "icos_settings": {
+    "node_reward_type": "type3.1",
+    "mgmt_mac": "00:00:00:00:00:01",
+    "deployment_environment": "mainnet",
+    "nns_urls": [
+      "https://icp-api.io/",
+      "https://icp0.io/",
+      "https://ic0.app/"
+    ],
+    "node_operator_private_key": null,
+    "enable_trusted_execution_environment": false,
+    "use_ssh_authorized_keys": false,
+    "icos_dev_settings": {}
+  },
+  "guestos_settings": {
+    "guestos_dev_settings": {
+      "backup_spool": null,
+      "malicious_behavior": null,
+      "query_stats_epoch_length": null,
+      "bitcoind_addr": null,
+      "dogecoind_addr": null,
+      "jaeger_addr": null,
+      "socks_proxy": null,
+      "hostname": null,
+      "generate_ic_boundary_tls_cert": null,
+      "ic_boundary_tls_cert": null,
+      "extra_api_boundary_node_trust_anchors_pem": null,
+      "nns_pub_key_override": null
+    }
+  },
+  "guest_vm_type": "default",
+  "upgrade_config": {
+    "peer_guest_vm_address": "2a00:fb01:400:200:6801:95ff:fed7:d475"
+  },
+  "trusted_execution_environment_config": {
+    "sev_cert_chain_pem": "-----BEGIN CERTIFICATE----------END CERTIFICATE-----"
+  },
+  "recovery_config": null
+}

rs/ic_os/config/types/compatibility_tests/fixtures/hostos_v1.16.0.json

@@ -0,0 +1,55 @@
+{
+  "config_version": "1.16.0",
+  "network_settings": {
+    "ipv6_config": {
+      "Fixed": {
+        "address": "2a00:fb01:400:200::1/64",
+        "gateway": "2a00:fb01:400:200::1"
+      }
+    },
+    "ipv4_config": {
+      "address": "192.168.1.1",
+      "gateway": "192.168.1.254",
+      "prefix_length": 24
+    },
+    "domain_name": "ic.test"
+  },
+  "icos_settings": {
+    "node_reward_type": "type3.1",
+    "mgmt_mac": "00:00:00:00:00:01",
+    "deployment_environment": "mainnet",
+    "nns_urls": [
+      "https://icp-api.io/",
+      "https://icp0.io/",
+      "https://ic0.app/"
+    ],
+    "node_operator_private_key": null,
+    "enable_trusted_execution_environment": false,
+    "use_ssh_authorized_keys": false,
+    "icos_dev_settings": {}
+  },
+  "hostos_settings": {
+    "hostos_dev_settings": {
+      "vm_memory": 16,
+      "vm_cpu": "kvm",
+      "vm_nr_of_vcpus": 64
+    },
+    "verbose": false
+  },
+  "guestos_settings": {
+    "guestos_dev_settings": {
+      "backup_spool": null,
+      "malicious_behavior": null,
+      "query_stats_epoch_length": null,
+      "bitcoind_addr": null,
+      "dogecoind_addr": null,
+      "jaeger_addr": null,
+      "socks_proxy": null,
+      "hostname": null,
+      "generate_ic_boundary_tls_cert": null,
+      "ic_boundary_tls_cert": null,
+      "extra_api_boundary_node_trust_anchors_pem": null,
+      "nns_pub_key_override": null
+    }
+  }
+}

rs/ic_os/config/types/compatibility_tests/src/fixture.rs

@@ -66,8 +66,14 @@ fn generate_hostos_config() -> HostOSConfig {
         mgmt_mac: "00:00:00:00:00:01".parse().unwrap(),
         deployment_environment: DeploymentEnvironment::Mainnet,
         enable_trusted_execution_environment: false,
+        // One `Url` per entry. Passing the whole comma-separated list to
+        // `Url::parse` yields a single `Url` whose host is `icp-api.io,https`
+        // and whose path is the rest, which re-serialises to the malformed
+        // `https://icp-api.io,https//icp0.io,https://ic0.app`.
         nns_urls: vec![
-            url::Url::parse("https://icp-api.io,https://icp0.io,https://ic0.app").unwrap(),
+            url::Url::parse("https://icp-api.io").unwrap(),
+            url::Url::parse("https://icp0.io").unwrap(),
+            url::Url::parse("https://ic0.app").unwrap(),
         ],
         node_operator_private_key: None,
         use_ssh_authorized_keys: false,

rs/ic_os/config/types/src/lib.rs

@@ -41,7 +41,7 @@ use std::str::FromStr;
 use strum::{Display, EnumString};
 use url::Url;
 
-pub const CONFIG_VERSION: &str = "1.15.0";
+pub const CONFIG_VERSION: &str = "1.16.0";
 
 /// List of field paths that have been removed and should not be reused.
 pub static RESERVED_FIELD_PATHS: &[&str] = &[
@@ -249,6 +249,16 @@ pub struct GuestOSDevSettings {
     /// Pre-generated TLS certificate and key for ic-boundary.
     #[serde(default)]
     pub ic_boundary_tls_cert: Option<IcBoundaryTlsCert>,
+    /// PEM-encoded certificates to trust, in addition to the public roots
+    /// compiled into the replica, when connecting to an API boundary node.
+    ///
+    /// A cloud engine subnet fetches its NNS delegation from an API boundary
+    /// node over TLS. In a testnet that node's certificate comes from a
+    /// throw-away CA rather than a public one, so that CA has to be handed to
+    /// the replica. Left unset in production, where the public roots are the
+    /// only trust anchors. To be used in system tests only.
+    #[serde(default)]
+    pub extra_api_boundary_node_trust_anchors_pem: Option<String>,
     /// PEM-encoded NNS public key.
     /// Overrides the hardcoded NNS public key on the rootfs.
     pub nns_pub_key_override: Option<String>,

rs/ic_os/networking/network/BUILD.bazel

@@ -10,6 +10,10 @@ rust_library(
     aliases = {},
     crate_name = "network",
     proc_macro_deps = [],
+    visibility = [
+        "//rs:ic-os-pkg",
+        "//rs:system-tests-pkg",
+    ],
     deps = [
         # Keep sorted.
         "//rs/ic_os/config/types:config_types",

rs/ic_os/networking/network/src/systemd.rs

@@ -12,6 +12,27 @@ use macaddr::MacAddr6;
 
 pub static DEFAULT_SYSTEMD_NETWORK_DIR: &str = "/run/systemd/network";
 
+/// The IPv6 name servers GuestOS is configured with (Cloudflare and Google).
+///
+/// Kept in sync with [`IPV6_NAME_SERVER_NETWORKD_CONTENTS`] by a unit test.
+///
+/// The system-test local backend depends on this being *the* set of addresses a
+/// GuestOS node sends its DNS queries to: it assigns them to the test group's
+/// bridge inside its isolated network namespace and answers on them, which is
+/// how nodes get a resolver without any node-side configuration. See
+/// `LocalBackend::create_group` in
+/// `rs/tests/driver/src/driver/local_backend.rs`.
+pub const IPV6_NAME_SERVERS: [Ipv6Addr; 4] = [
+    // 2606:4700:4700::1111
+    Ipv6Addr::new(0x2606, 0x4700, 0x4700, 0, 0, 0, 0, 0x1111),
+    // 2606:4700:4700::1001
+    Ipv6Addr::new(0x2606, 0x4700, 0x4700, 0, 0, 0, 0, 0x1001),
+    // 2001:4860:4860::8888
+    Ipv6Addr::new(0x2001, 0x4860, 0x4860, 0, 0, 0, 0, 0x8888),
+    // 2001:4860:4860::8844
+    Ipv6Addr::new(0x2001, 0x4860, 0x4860, 0, 0, 0, 0, 0x8844),
+];
+
 pub const IPV6_NAME_SERVER_NETWORKD_CONTENTS: &str = r#"
 DNS=2606:4700:4700::1111
 DNS=2606:4700:4700::1001
@@ -157,3 +178,18 @@ fn generate_and_write_systemd_files(
 
     Ok(())
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_ipv6_name_servers_match_networkd_contents() {
+        let rendered: String = IPV6_NAME_SERVERS
+            .iter()
+            .map(|name_server| format!("DNS={name_server}\n"))
+            .collect();
+
+        assert_eq!(IPV6_NAME_SERVER_NETWORKD_CONTENTS, format!("\n{rendered}"));
+    }
+}

rs/orchestrator/src/firewall.rs

@@ -1404,6 +1404,7 @@ mod tests {
             domain_name: "".to_string(),
             node_reward_type: "".to_string(),
             malicious_behavior: "null".to_string(),
+            extra_api_boundary_node_trust_anchors_pem: "null".to_string(),
         };
 
         let ic_json = generate_ic_config::render_ic_config(template)

rs/tests/consensus/orchestrator/BUILD.bazel

@@ -151,21 +151,6 @@ system_test(
 
 system_test_nns(
     name = "standard_engine_replica_version_test",
-    # TODO: Remove this once PR 11208 is in master.
-    # Why: This test needs an API Boundary Node, and those do not work
-    # when you do `bazel test //rs/tests:widget_test_local`. This test
-    # only works, when run in Farm. This blocks `_local`.
-    #
-    # Ok, but why is API BN needed: Cloud Engines.
-    #
-    # Ok, but why do Cloud Engines need API BNs? They are not allowed
-    # to talk NATIVELY to canisters that live in other subnets, but
-    # they need to talk to the Registry canister, which lives in the NNS
-    # subnet.
-    #
-    # To satisfy these conflicting requirements, Cloud Engines talk to
-    # Registry via API BNs.
-    backend = "farm",
     # 1 API BN + 1 System + 2 * 4 Cloud Engine = 10 IC Node VMs * 6 vCPUs.
     cpus = MIN_LOCAL_CPUS + 10 * DEFAULT_VCPUS_PER_VM,
     # TODO: Re-enable this once BOTH of these pins in