diff --git a/server/configuration.md b/server/configuration.md index 57e6289a..854a4d88 100644 --- a/server/configuration.md +++ b/server/configuration.md @@ -173,6 +173,11 @@ Configures the **egress sidecar** image and enforcement mode. The server only at |-----|------|---------|-------------| | `image` | string \| omitted | `null` | OCI image for the egress sidecar. **Required in config** when clients send **`networkPolicy`** (create request). | | `mode` | string | `"dns"` | Passed to the sidecar as `OPENSANDBOX_EGRESS_MODE`. Values: **`dns`** — DNS-proxy-based enforcement (CIDR/static IP rules **not** enforced); **`dns+nft`** — adds nftables where available so **CIDR/IP** rules can be enforced. | +| `disable_ipv6` | bool | `true` | IPv6 egress is incomplete (especially on Kubernetes). **Default on**; set `false` only when you want IPv6 left up in the netns. Details in [IPv6 and egress](#ipv6-and-egress) below. | + +### IPv6 and egress + +OpenSandbox egress does **not** treat IPv6 as a first-class, fully covered path—gaps show up most often under **`runtime.type = "kubernetes"`** (pod networking, CNI). The default **`disable_ipv6 = true`** matches the usual need on **dual-stack** CNI: do not rely on incomplete IPv6 egress. Set **`false`** when the cluster is effectively **IPv4-only** and you deliberately want IPv6 enabled in the sandbox network namespace, or when you accept those gaps for experiments. **Docker notes:** diff --git a/server/opensandbox_server/config.py b/server/opensandbox_server/config.py index 0abc012e..b85947a2 100644 --- a/server/opensandbox_server/config.py +++ b/server/opensandbox_server/config.py @@ -425,6 +425,14 @@ class EgressConfig(BaseModel): default=EGRESS_MODE_DNS, description="Egress enforcement passed to the sidecar as OPENSANDBOX_EGRESS_MODE (dns or dns+nft).", ) + disable_ipv6: bool = Field( + default=True, + description=( + "Default true: egress IPv6 support is incomplete, especially on Kubernetes runtime. " + "Set false only if you intentionally leave IPv6 enabled in the sandbox netns " + "(e.g. IPv4-only CNI or experimenting with IPv6 egress despite gaps)." + ), + ) class RuntimeConfig(BaseModel): diff --git a/server/opensandbox_server/examples/example.config.k8s.toml b/server/opensandbox_server/examples/example.config.k8s.toml index c6eddcc4..976c1b54 100644 --- a/server/opensandbox_server/examples/example.config.k8s.toml +++ b/server/opensandbox_server/examples/example.config.k8s.toml @@ -81,3 +81,5 @@ mode = "direct" image = "opensandbox/egress:v1.0.4" # Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS). mode = "dns" +# Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns (see server/configuration.md). +# disable_ipv6 = false diff --git a/server/opensandbox_server/examples/example.config.k8s.zh.toml b/server/opensandbox_server/examples/example.config.k8s.zh.toml index 74c1c30f..e4b71f14 100644 --- a/server/opensandbox_server/examples/example.config.k8s.zh.toml +++ b/server/opensandbox_server/examples/example.config.k8s.zh.toml @@ -82,3 +82,5 @@ mode = "direct" image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.4" # Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS). mode = "dns" +# Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns (see server/configuration.md). +# disable_ipv6 = false diff --git a/server/opensandbox_server/examples/example.config.toml b/server/opensandbox_server/examples/example.config.toml index e4f0c7cd..62885753 100644 --- a/server/opensandbox_server/examples/example.config.toml +++ b/server/opensandbox_server/examples/example.config.toml @@ -48,6 +48,8 @@ execd_image = "opensandbox/execd:v1.0.9" image = "opensandbox/egress:v1.0.4" # Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS). mode = "dns" +# Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns. +# disable_ipv6 = false [storage] # Volume and storage configuration diff --git a/server/opensandbox_server/examples/example.config.zh.toml b/server/opensandbox_server/examples/example.config.zh.toml index f217d039..a7d5c1dd 100644 --- a/server/opensandbox_server/examples/example.config.zh.toml +++ b/server/opensandbox_server/examples/example.config.zh.toml @@ -45,6 +45,8 @@ execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.4" # Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS). mode = "dns" +# Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns. +# disable_ipv6 = false [storage] # 卷存储配置 diff --git a/server/opensandbox_server/services/docker.py b/server/opensandbox_server/services/docker.py index 89e3c980..ab74e516 100644 --- a/server/opensandbox_server/services/docker.py +++ b/server/opensandbox_server/services/docker.py @@ -2033,13 +2033,14 @@ def _start_egress_sidecar( "44772": ("0.0.0.0", host_execd_port), "8080": ("0.0.0.0", host_http_port), }, - # FIXME(Pangjiping): Disable IPv6 in the shared namespace to keep policy enforcement consistent. - "sysctls": { + } + if self.app_config.egress.disable_ipv6: + # Optional: disable IPv6 in the shared namespace when egress.disable_ipv6 is set. + sidecar_host_config_kwargs["sysctls"] = { "net.ipv6.conf.all.disable_ipv6": 1, "net.ipv6.conf.default.disable_ipv6": 1, "net.ipv6.conf.lo.disable_ipv6": 1, - }, - } + } sidecar_host_config = self.docker_client.api.create_host_config( **sidecar_host_config_kwargs diff --git a/server/opensandbox_server/services/k8s/agent_sandbox_provider.py b/server/opensandbox_server/services/k8s/agent_sandbox_provider.py index 08692729..c7bad7c1 100644 --- a/server/opensandbox_server/services/k8s/agent_sandbox_provider.py +++ b/server/opensandbox_server/services/k8s/agent_sandbox_provider.py @@ -108,6 +108,12 @@ def __init__( self.resolver.get_k8s_runtime_class() if self.resolver else None ) + self.egress_disable_ipv6 = ( + bool(app_config.egress.disable_ipv6) + if app_config and app_config.egress is not None + else False + ) + def _resource_name(self, sandbox_id: str) -> str: return _to_dns1035_label(sandbox_id, prefix="sandbox") @@ -225,7 +231,11 @@ def _build_pod_spec( egress_mode: str = EGRESS_MODE_DNS, ) -> Dict[str, Any]: """Build pod spec dict for the Sandbox CRD.""" - disable_ipv6_for_egress = network_policy is not None and egress_image is not None + disable_ipv6_for_egress = ( + network_policy is not None + and egress_image is not None + and self.egress_disable_ipv6 + ) init_container = self._build_execd_init_container( execd_image, disable_ipv6_for_egress=disable_ipv6_for_egress ) @@ -273,7 +283,10 @@ def _build_execd_init_container( *, disable_ipv6_for_egress: bool = False, ) -> V1Container: - """Build init container that copies execd binary to the shared volume.""" + """Build init container that copies execd binary to the shared volume. + + ``disable_ipv6_for_egress`` is True only when ``egress.disable_ipv6`` is set and egress is used. + """ script = ( "cp ./execd /opt/opensandbox/bin/execd && " "cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && " diff --git a/server/opensandbox_server/services/k8s/batchsandbox_provider.py b/server/opensandbox_server/services/k8s/batchsandbox_provider.py index c7a49ccd..9995f450 100644 --- a/server/opensandbox_server/services/k8s/batchsandbox_provider.py +++ b/server/opensandbox_server/services/k8s/batchsandbox_provider.py @@ -97,6 +97,12 @@ def __init__( # Template manager self.template_manager = BatchSandboxTemplateManager(template_file_path) + self.egress_disable_ipv6 = ( + bool(app_config.egress.disable_ipv6) + if app_config and app_config.egress is not None + else False + ) + def supports_image_auth(self) -> bool: """BatchSandbox supports image pull auth via imagePullSecrets injection.""" return True @@ -185,7 +191,11 @@ def create_workload( extra_volumes, extra_mounts = self._extract_template_pod_extras() # Build init container for execd installation - disable_ipv6_for_egress = network_policy is not None and egress_image is not None + disable_ipv6_for_egress = ( + network_policy is not None + and egress_image is not None + and self.egress_disable_ipv6 + ) init_container = self._build_execd_init_container( execd_image, disable_ipv6_for_egress=disable_ipv6_for_egress ) @@ -517,8 +527,8 @@ def _build_execd_init_container( Args: execd_image: execd container image - disable_ipv6_for_egress: When True, disable IPv6 in the Pod netns first - (privileged) then install binaries; used with egress sidecar. + disable_ipv6_for_egress: When True (``egress.disable_ipv6`` in server config), + disable IPv6 in the Pod netns first (privileged) then install binaries. Returns: V1Container: Init container spec diff --git a/server/opensandbox_server/services/k8s/egress_helper.py b/server/opensandbox_server/services/k8s/egress_helper.py index d0c82743..f20c6cfa 100644 --- a/server/opensandbox_server/services/k8s/egress_helper.py +++ b/server/opensandbox_server/services/k8s/egress_helper.py @@ -33,7 +33,7 @@ def prep_execd_init_for_egress(exec_install_script: str) -> tuple[str, Dict[str, Any]]: """ - Prepare execd init when an egress sidecar is used: disable IPv6 in the Pod netns, then install. + Prepare execd init when ``egress.disable_ipv6`` is true: disable IPv6 in the Pod netns, then install. Writes ``/proc/sys/.../disable_ipv6`` (no ``sysctl`` binary required). The returned security context dict must be applied to the execd init container (typically via @@ -77,8 +77,8 @@ def apply_egress_to_spec( egress_mode: str = EGRESS_MODE_DNS, ) -> None: """ - Append the egress sidecar to ``containers``. IPv6 is handled in execd init - (``prep_execd_init_for_egress``); Pod-level sysctls are not modified. + Append the egress sidecar to ``containers``. When ``egress.disable_ipv6`` is enabled, + IPv6 is handled in execd init (``prep_execd_init_for_egress``); Pod-level sysctls are not modified. """ if not network_policy or not egress_image: return diff --git a/server/tests/k8s/test_agent_sandbox_provider.py b/server/tests/k8s/test_agent_sandbox_provider.py index 682b1757..c50441a5 100644 --- a/server/tests/k8s/test_agent_sandbox_provider.py +++ b/server/tests/k8s/test_agent_sandbox_provider.py @@ -29,6 +29,7 @@ AgentSandboxRuntimeConfig, EGRESS_MODE_DNS, EGRESS_MODE_DNS_NFT, + EgressConfig, ExecdInitResources, KubernetesRuntimeConfig, RuntimeConfig, @@ -38,7 +39,12 @@ from opensandbox_server.services.constants import OPENSANDBOX_EGRESS_TOKEN -def _app_config(shutdown_policy: str = "Delete", service_account: str | None = None, execd_init_resources: ExecdInitResources | None = None) -> AppConfig: +def _app_config( + shutdown_policy: str = "Delete", + service_account: str | None = None, + execd_init_resources: ExecdInitResources | None = None, + egress: EgressConfig | None = None, +) -> AppConfig: """Build an AppConfig for AgentSandboxProvider tests.""" return AppConfig( runtime=RuntimeConfig(type="kubernetes", execd_image="execd:test"), @@ -49,6 +55,7 @@ def _app_config(shutdown_policy: str = "Delete", service_account: str | None = N execd_init_resources=execd_init_resources, ), agent_sandbox=AgentSandboxRuntimeConfig(shutdown_policy=shutdown_policy), + egress=egress, ) @@ -498,7 +505,10 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) """ Test case: Verify egress sidecar is added when network_policy is provided """ - provider = AgentSandboxProvider(mock_k8s_client) + provider = AgentSandboxProvider( + mock_k8s_client, + _app_config(egress=EgressConfig()), + ) mock_k8s_client.create_custom_object.return_value = { "metadata": {"name": "test-id", "uid": "test-uid"} } @@ -614,7 +624,10 @@ def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client): assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS_NFT def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self, mock_k8s_client): - provider = AgentSandboxProvider(mock_k8s_client) + provider = AgentSandboxProvider( + mock_k8s_client, + _app_config(egress=EgressConfig()), + ) mock_k8s_client.create_custom_object.return_value = { "metadata": {"name": "test-id", "uid": "test-uid"} } @@ -650,6 +663,42 @@ def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self, assert execd_init["name"] == "execd-installer" assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in execd_init["args"][0] + def test_create_workload_with_egress_skips_ipv6_disable_when_not_configured(self, mock_k8s_client): + """With ``egress.disable_ipv6`` false, execd init stays unprivileged without sysctl writes.""" + provider = AgentSandboxProvider( + mock_k8s_client, + _app_config(egress=EgressConfig(disable_ipv6=False)), + ) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "test-id", "uid": "test-uid"} + } + + network_policy = NetworkPolicy( + default_action="deny", + egress=[NetworkRule(action="allow", target="example.com")], + ) + + provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri="python:3.11"), + entrypoint=["/bin/bash"], + env={}, + resource_limits={}, + labels={}, + expires_at=None, + execd_image="execd:latest", + network_policy=network_policy, + egress_image="opensandbox/egress:v1.0.3", + ) + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + pod_spec = body["spec"]["podTemplate"]["spec"] + execd_init = pod_spec["initContainers"][0] + assert execd_init["name"] == "execd-installer" + assert "securityContext" not in execd_init + assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" not in execd_init["args"][0] + def test_create_workload_with_network_policy_drops_net_admin_from_main_container(self, mock_k8s_client): """ Test case: Verify main container drops NET_ADMIN when network_policy is enabled diff --git a/server/tests/k8s/test_batchsandbox_provider.py b/server/tests/k8s/test_batchsandbox_provider.py index 5fadaedb..ae4695ad 100644 --- a/server/tests/k8s/test_batchsandbox_provider.py +++ b/server/tests/k8s/test_batchsandbox_provider.py @@ -26,6 +26,7 @@ AppConfig, EGRESS_MODE_DNS, EGRESS_MODE_DNS_NFT, + EgressConfig, ExecdInitResources, KubernetesRuntimeConfig, RuntimeConfig, @@ -59,6 +60,15 @@ def _app_config_with_execd_resources(execd_init_resources: ExecdInitResources) - ) +def _app_config_with_egress_disable_ipv6(disable_ipv6: bool = True) -> AppConfig: + """Build an AppConfig with ``egress.disable_ipv6`` set (privileged execd init when egress is used).""" + return AppConfig( + runtime=RuntimeConfig(type="kubernetes", execd_image="execd:test"), + kubernetes=KubernetesRuntimeConfig(namespace="test-ns"), + egress=EgressConfig(disable_ipv6=disable_ipv6), + ) + + class TestBatchSandboxProvider: """BatchSandboxProvider unit tests""" @@ -1203,7 +1213,10 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client) """ Test case: Verify egress sidecar is added when network_policy is provided """ - provider = BatchSandboxProvider(mock_k8s_client) + provider = BatchSandboxProvider( + mock_k8s_client, + _app_config_with_egress_disable_ipv6(), + ) mock_k8s_client.create_custom_object.return_value = { "metadata": {"name": "test-id", "uid": "test-uid"} } @@ -1320,7 +1333,10 @@ def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client): def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self, mock_k8s_client): """IPv6 all.disable is applied in privileged execd init, not Pod sysctls.""" - provider = BatchSandboxProvider(mock_k8s_client) + provider = BatchSandboxProvider( + mock_k8s_client, + _app_config_with_egress_disable_ipv6(), + ) mock_k8s_client.create_custom_object.return_value = { "metadata": {"name": "test-id", "uid": "test-uid"} } @@ -1356,6 +1372,42 @@ def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self, assert execd_init["name"] == "execd-installer" assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in execd_init["args"][0] + def test_create_workload_with_egress_skips_ipv6_disable_when_not_configured(self, mock_k8s_client): + """With ``egress.disable_ipv6`` false, execd init is not privileged and does not write disable_ipv6.""" + provider = BatchSandboxProvider( + mock_k8s_client, + _app_config_with_egress_disable_ipv6(False), + ) + mock_k8s_client.create_custom_object.return_value = { + "metadata": {"name": "test-id", "uid": "test-uid"} + } + + network_policy = NetworkPolicy( + default_action="deny", + egress=[NetworkRule(action="allow", target="example.com")], + ) + + provider.create_workload( + sandbox_id="test-id", + namespace="test-ns", + image_spec=ImageSpec(uri="python:3.11"), + entrypoint=["/bin/bash"], + env={}, + resource_limits={}, + labels={}, + expires_at=None, + execd_image="execd:latest", + network_policy=network_policy, + egress_image="opensandbox/egress:v1.0.3", + ) + + body = mock_k8s_client.create_custom_object.call_args.kwargs["body"] + pod_spec = body["spec"]["template"]["spec"] + execd_init = pod_spec["initContainers"][0] + assert execd_init["name"] == "execd-installer" + assert "securityContext" not in execd_init + assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" not in execd_init["args"][0] + def test_create_workload_with_network_policy_drops_net_admin_from_main_container(self, mock_k8s_client): """ Test case: Verify main container drops NET_ADMIN when network_policy is enabled diff --git a/server/tests/k8s/test_egress_helper.py b/server/tests/k8s/test_egress_helper.py index 7a4d3cac..d08ba113 100644 --- a/server/tests/k8s/test_egress_helper.py +++ b/server/tests/k8s/test_egress_helper.py @@ -175,7 +175,7 @@ def test_handles_missing_default_action(self): assert "egress" in policy_dict def test_security_context_adds_net_admin_not_privileged(self): - """Egress sidecar uses NET_ADMIN only (IPv6 is disabled in execd init when egress is on).""" + """Egress sidecar uses NET_ADMIN only (IPv6 sysctl is optional via ``egress.disable_ipv6``).""" egress_image = "opensandbox/egress:v1.0.4" network_policy = NetworkPolicy( default_action="deny", diff --git a/server/tests/test_config.py b/server/tests/test_config.py index 80922d3c..4125ff2a 100644 --- a/server/tests/test_config.py +++ b/server/tests/test_config.py @@ -692,6 +692,8 @@ def test_kubernetes_runtime_with_firecracker_is_valid(): def test_egress_config_mode_literal(): - assert EgressConfig(image="opensandbox/egress:v1").mode == EGRESS_MODE_DNS + base = EgressConfig(image="opensandbox/egress:v1") + assert base.mode == EGRESS_MODE_DNS + assert base.disable_ipv6 is True cfg = EgressConfig(image="opensandbox/egress:v1", mode=EGRESS_MODE_DNS_NFT) assert cfg.mode == EGRESS_MODE_DNS_NFT diff --git a/server/tests/test_docker_service.py b/server/tests/test_docker_service.py index 1249dfb2..6e746e92 100644 --- a/server/tests/test_docker_service.py +++ b/server/tests/test_docker_service.py @@ -666,6 +666,63 @@ def host_cfg_side_effect(**kwargs): mock_client.api.remove_container.assert_called_once_with("sidecar-id", force=True) +@patch("opensandbox_server.services.docker.docker") +def test_egress_sidecar_host_config_sysctls_only_when_egress_disable_ipv6(mock_docker): + mock_client = MagicMock() + mock_client.containers.list.return_value = [] + + def host_cfg_side_effect(**kwargs): + return kwargs + + mock_client.api.create_host_config.side_effect = host_cfg_side_effect + mock_client.api.create_container.return_value = {"Id": "sidecar-id"} + mock_client.containers.get.return_value = MagicMock() + mock_docker.from_env.return_value = mock_client + + cfg = _app_config() + cfg.docker.network_mode = "bridge" + cfg.egress = EgressConfig(image="egress:latest", disable_ipv6=False) + service = DockerSandboxService(config=cfg) + + with ( + patch.object(service, "_ensure_image_available"), + patch.object(service, "_docker_operation") as mock_op, + ): + mock_op.return_value.__enter__.return_value = None + mock_op.return_value.__exit__.return_value = None + service._start_egress_sidecar( + "sandbox-id", + NetworkPolicy(defaultAction="deny", egress=[]), + egress_token="egress-token", + host_execd_port=44772, + host_http_port=8080, + ) + + hc_kwargs = mock_client.api.create_host_config.call_args.kwargs + assert "sysctls" not in hc_kwargs + + cfg.egress = EgressConfig(image="egress:latest", disable_ipv6=True) + service2 = DockerSandboxService(config=cfg) + mock_client.api.create_host_config.reset_mock() + + with ( + patch.object(service2, "_ensure_image_available"), + patch.object(service2, "_docker_operation") as mock_op2, + ): + mock_op2.return_value.__enter__.return_value = None + mock_op2.return_value.__exit__.return_value = None + service2._start_egress_sidecar( + "sandbox-id", + NetworkPolicy(defaultAction="deny", egress=[]), + egress_token="egress-token", + host_execd_port=44772, + host_http_port=8080, + ) + + hc2 = mock_client.api.create_host_config.call_args.kwargs + assert hc2["sysctls"]["net.ipv6.conf.all.disable_ipv6"] == 1 + + def test_expire_cleans_sidecar(): service = DockerSandboxService(config=_app_config()) mock_container = MagicMock()