Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions server/configuration.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,11 @@ Configures the **egress sidecar** image and enforcement mode. The server only at
|-----|------|---------|-------------|
| `image` | string \| omitted | `null` | OCI image for the egress sidecar. **Required in config** when clients send **`networkPolicy`** (create request). |
| `mode` | string | `"dns"` | Passed to the sidecar as `OPENSANDBOX_EGRESS_MODE`. Values: **`dns`** — DNS-proxy-based enforcement (CIDR/static IP rules **not** enforced); **`dns+nft`** — adds nftables where available so **CIDR/IP** rules can be enforced. |
| `disable_ipv6` | bool | `true` | IPv6 egress is incomplete (especially on Kubernetes). **Default on**; set `false` only when you want IPv6 left up in the netns. Details in [IPv6 and egress](#ipv6-and-egress) below. |

### IPv6 and egress

OpenSandbox egress does **not** treat IPv6 as a first-class, fully covered path—gaps show up most often under **`runtime.type = "kubernetes"`** (pod networking, CNI). The default **`disable_ipv6 = true`** matches the usual need on **dual-stack** CNI: do not rely on incomplete IPv6 egress. Set **`false`** when the cluster is effectively **IPv4-only** and you deliberately want IPv6 enabled in the sandbox network namespace, or when you accept those gaps for experiments.

**Docker notes:**

Expand Down
8 changes: 8 additions & 0 deletions server/opensandbox_server/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -425,6 +425,14 @@ class EgressConfig(BaseModel):
default=EGRESS_MODE_DNS,
description="Egress enforcement passed to the sidecar as OPENSANDBOX_EGRESS_MODE (dns or dns+nft).",
)
disable_ipv6: bool = Field(
default=True,
description=(
"Default true: egress IPv6 support is incomplete, especially on Kubernetes runtime. "
"Set false only if you intentionally leave IPv6 enabled in the sandbox netns "
"(e.g. IPv4-only CNI or experimenting with IPv6 egress despite gaps)."
),
)


class RuntimeConfig(BaseModel):
Expand Down
2 changes: 2 additions & 0 deletions server/opensandbox_server/examples/example.config.k8s.toml
Original file line number Diff line number Diff line change
Expand Up @@ -81,3 +81,5 @@ mode = "direct"
image = "opensandbox/egress:v1.0.4"
# Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS).
mode = "dns"
# Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns (see server/configuration.md).
# disable_ipv6 = false
Original file line number Diff line number Diff line change
Expand Up @@ -82,3 +82,5 @@ mode = "direct"
image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.4"
# Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS).
mode = "dns"
# Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns (see server/configuration.md).
# disable_ipv6 = false
2 changes: 2 additions & 0 deletions server/opensandbox_server/examples/example.config.toml
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ execd_image = "opensandbox/execd:v1.0.9"
image = "opensandbox/egress:v1.0.4"
# Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS).
mode = "dns"
# Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns.
# disable_ipv6 = false

[storage]
# Volume and storage configuration
Expand Down
2 changes: 2 additions & 0 deletions server/opensandbox_server/examples/example.config.zh.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,8 @@ execd_image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/execd
image = "sandbox-registry.cn-zhangjiakou.cr.aliyuncs.com/opensandbox/egress:v1.0.4"
# Enforcement: "dns" (DNS proxy only) or "dns+nft" (nftables + DNS).
mode = "dns"
# Default is true (recommended for dual-stack CNI). Set false only if you need IPv6 in the netns.
# disable_ipv6 = false

[storage]
# 卷存储配置
Expand Down
9 changes: 5 additions & 4 deletions server/opensandbox_server/services/docker.py
Original file line number Diff line number Diff line change
Expand Up @@ -2033,13 +2033,14 @@ def _start_egress_sidecar(
"44772": ("0.0.0.0", host_execd_port),
"8080": ("0.0.0.0", host_http_port),
},
# FIXME(Pangjiping): Disable IPv6 in the shared namespace to keep policy enforcement consistent.
"sysctls": {
}
if self.app_config.egress.disable_ipv6:
# Optional: disable IPv6 in the shared namespace when egress.disable_ipv6 is set.
sidecar_host_config_kwargs["sysctls"] = {
"net.ipv6.conf.all.disable_ipv6": 1,
"net.ipv6.conf.default.disable_ipv6": 1,
"net.ipv6.conf.lo.disable_ipv6": 1,
},
}
}

sidecar_host_config = self.docker_client.api.create_host_config(
**sidecar_host_config_kwargs
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,12 @@ def __init__(
self.resolver.get_k8s_runtime_class() if self.resolver else None
)

self.egress_disable_ipv6 = (
bool(app_config.egress.disable_ipv6)
if app_config and app_config.egress is not None
else False
)

def _resource_name(self, sandbox_id: str) -> str:
return _to_dns1035_label(sandbox_id, prefix="sandbox")

Expand Down Expand Up @@ -225,7 +231,11 @@ def _build_pod_spec(
egress_mode: str = EGRESS_MODE_DNS,
) -> Dict[str, Any]:
"""Build pod spec dict for the Sandbox CRD."""
disable_ipv6_for_egress = network_policy is not None and egress_image is not None
disable_ipv6_for_egress = (
network_policy is not None
and egress_image is not None
and self.egress_disable_ipv6
)
init_container = self._build_execd_init_container(
execd_image, disable_ipv6_for_egress=disable_ipv6_for_egress
)
Expand Down Expand Up @@ -273,7 +283,10 @@ def _build_execd_init_container(
*,
disable_ipv6_for_egress: bool = False,
) -> V1Container:
"""Build init container that copies execd binary to the shared volume."""
"""Build init container that copies execd binary to the shared volume.

``disable_ipv6_for_egress`` is True only when ``egress.disable_ipv6`` is set and egress is used.
"""
script = (
"cp ./execd /opt/opensandbox/bin/execd && "
"cp ./bootstrap.sh /opt/opensandbox/bin/bootstrap.sh && "
Expand Down
16 changes: 13 additions & 3 deletions server/opensandbox_server/services/k8s/batchsandbox_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,12 @@ def __init__(
# Template manager
self.template_manager = BatchSandboxTemplateManager(template_file_path)

self.egress_disable_ipv6 = (
bool(app_config.egress.disable_ipv6)
if app_config and app_config.egress is not None
else False
)

def supports_image_auth(self) -> bool:
"""BatchSandbox supports image pull auth via imagePullSecrets injection."""
return True
Expand Down Expand Up @@ -185,7 +191,11 @@ def create_workload(
extra_volumes, extra_mounts = self._extract_template_pod_extras()

# Build init container for execd installation
disable_ipv6_for_egress = network_policy is not None and egress_image is not None
disable_ipv6_for_egress = (
network_policy is not None
and egress_image is not None
and self.egress_disable_ipv6
)
init_container = self._build_execd_init_container(
execd_image, disable_ipv6_for_egress=disable_ipv6_for_egress
)
Expand Down Expand Up @@ -517,8 +527,8 @@ def _build_execd_init_container(

Args:
execd_image: execd container image
disable_ipv6_for_egress: When True, disable IPv6 in the Pod netns first
(privileged) then install binaries; used with egress sidecar.
disable_ipv6_for_egress: When True (``egress.disable_ipv6`` in server config),
disable IPv6 in the Pod netns first (privileged) then install binaries.

Returns:
V1Container: Init container spec
Expand Down
6 changes: 3 additions & 3 deletions server/opensandbox_server/services/k8s/egress_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@

def prep_execd_init_for_egress(exec_install_script: str) -> tuple[str, Dict[str, Any]]:
"""
Prepare execd init when an egress sidecar is used: disable IPv6 in the Pod netns, then install.
Prepare execd init when ``egress.disable_ipv6`` is true: disable IPv6 in the Pod netns, then install.

Writes ``/proc/sys/.../disable_ipv6`` (no ``sysctl`` binary required). The returned
security context dict must be applied to the execd init container (typically via
Expand Down Expand Up @@ -77,8 +77,8 @@ def apply_egress_to_spec(
egress_mode: str = EGRESS_MODE_DNS,
) -> None:
"""
Append the egress sidecar to ``containers``. IPv6 is handled in execd init
(``prep_execd_init_for_egress``); Pod-level sysctls are not modified.
Append the egress sidecar to ``containers``. When ``egress.disable_ipv6`` is enabled,
IPv6 is handled in execd init (``prep_execd_init_for_egress``); Pod-level sysctls are not modified.
"""
if not network_policy or not egress_image:
return
Expand Down
55 changes: 52 additions & 3 deletions server/tests/k8s/test_agent_sandbox_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
AgentSandboxRuntimeConfig,
EGRESS_MODE_DNS,
EGRESS_MODE_DNS_NFT,
EgressConfig,
ExecdInitResources,
KubernetesRuntimeConfig,
RuntimeConfig,
Expand All @@ -38,7 +39,12 @@
from opensandbox_server.services.constants import OPENSANDBOX_EGRESS_TOKEN


def _app_config(shutdown_policy: str = "Delete", service_account: str | None = None, execd_init_resources: ExecdInitResources | None = None) -> AppConfig:
def _app_config(
shutdown_policy: str = "Delete",
service_account: str | None = None,
execd_init_resources: ExecdInitResources | None = None,
egress: EgressConfig | None = None,
) -> AppConfig:
"""Build an AppConfig for AgentSandboxProvider tests."""
return AppConfig(
runtime=RuntimeConfig(type="kubernetes", execd_image="execd:test"),
Expand All @@ -49,6 +55,7 @@ def _app_config(shutdown_policy: str = "Delete", service_account: str | None = N
execd_init_resources=execd_init_resources,
),
agent_sandbox=AgentSandboxRuntimeConfig(shutdown_policy=shutdown_policy),
egress=egress,
)


Expand Down Expand Up @@ -498,7 +505,10 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client)
"""
Test case: Verify egress sidecar is added when network_policy is provided
"""
provider = AgentSandboxProvider(mock_k8s_client)
provider = AgentSandboxProvider(
mock_k8s_client,
_app_config(egress=EgressConfig()),
)
mock_k8s_client.create_custom_object.return_value = {
"metadata": {"name": "test-id", "uid": "test-uid"}
}
Expand Down Expand Up @@ -614,7 +624,10 @@ def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client):
assert env_vars["OPENSANDBOX_EGRESS_MODE"] == EGRESS_MODE_DNS_NFT

def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self, mock_k8s_client):
provider = AgentSandboxProvider(mock_k8s_client)
provider = AgentSandboxProvider(
mock_k8s_client,
_app_config(egress=EgressConfig()),
)
mock_k8s_client.create_custom_object.return_value = {
"metadata": {"name": "test-id", "uid": "test-uid"}
}
Expand Down Expand Up @@ -650,6 +663,42 @@ def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self,
assert execd_init["name"] == "execd-installer"
assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in execd_init["args"][0]

def test_create_workload_with_egress_skips_ipv6_disable_when_not_configured(self, mock_k8s_client):
"""With ``egress.disable_ipv6`` false, execd init stays unprivileged without sysctl writes."""
provider = AgentSandboxProvider(
mock_k8s_client,
_app_config(egress=EgressConfig(disable_ipv6=False)),
)
mock_k8s_client.create_custom_object.return_value = {
"metadata": {"name": "test-id", "uid": "test-uid"}
}

network_policy = NetworkPolicy(
default_action="deny",
egress=[NetworkRule(action="allow", target="example.com")],
)

provider.create_workload(
sandbox_id="test-id",
namespace="test-ns",
image_spec=ImageSpec(uri="python:3.11"),
entrypoint=["/bin/bash"],
env={},
resource_limits={},
labels={},
expires_at=None,
execd_image="execd:latest",
network_policy=network_policy,
egress_image="opensandbox/egress:v1.0.3",
)

body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
pod_spec = body["spec"]["podTemplate"]["spec"]
execd_init = pod_spec["initContainers"][0]
assert execd_init["name"] == "execd-installer"
assert "securityContext" not in execd_init
assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" not in execd_init["args"][0]

def test_create_workload_with_network_policy_drops_net_admin_from_main_container(self, mock_k8s_client):
"""
Test case: Verify main container drops NET_ADMIN when network_policy is enabled
Expand Down
56 changes: 54 additions & 2 deletions server/tests/k8s/test_batchsandbox_provider.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
AppConfig,
EGRESS_MODE_DNS,
EGRESS_MODE_DNS_NFT,
EgressConfig,
ExecdInitResources,
KubernetesRuntimeConfig,
RuntimeConfig,
Expand Down Expand Up @@ -59,6 +60,15 @@ def _app_config_with_execd_resources(execd_init_resources: ExecdInitResources) -
)


def _app_config_with_egress_disable_ipv6(disable_ipv6: bool = True) -> AppConfig:
"""Build an AppConfig with ``egress.disable_ipv6`` set (privileged execd init when egress is used)."""
return AppConfig(
runtime=RuntimeConfig(type="kubernetes", execd_image="execd:test"),
kubernetes=KubernetesRuntimeConfig(namespace="test-ns"),
egress=EgressConfig(disable_ipv6=disable_ipv6),
)


class TestBatchSandboxProvider:
"""BatchSandboxProvider unit tests"""

Expand Down Expand Up @@ -1203,7 +1213,10 @@ def test_create_workload_with_network_policy_adds_sidecar(self, mock_k8s_client)
"""
Test case: Verify egress sidecar is added when network_policy is provided
"""
provider = BatchSandboxProvider(mock_k8s_client)
provider = BatchSandboxProvider(
mock_k8s_client,
_app_config_with_egress_disable_ipv6(),
)
mock_k8s_client.create_custom_object.return_value = {
"metadata": {"name": "test-id", "uid": "test-uid"}
}
Expand Down Expand Up @@ -1320,7 +1333,10 @@ def test_create_workload_with_egress_mode_dns_nft(self, mock_k8s_client):

def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self, mock_k8s_client):
"""IPv6 all.disable is applied in privileged execd init, not Pod sysctls."""
provider = BatchSandboxProvider(mock_k8s_client)
provider = BatchSandboxProvider(
mock_k8s_client,
_app_config_with_egress_disable_ipv6(),
)
mock_k8s_client.create_custom_object.return_value = {
"metadata": {"name": "test-id", "uid": "test-uid"}
}
Expand Down Expand Up @@ -1356,6 +1372,42 @@ def test_create_workload_with_network_policy_does_not_add_pod_ipv6_sysctls(self,
assert execd_init["name"] == "execd-installer"
assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" in execd_init["args"][0]

def test_create_workload_with_egress_skips_ipv6_disable_when_not_configured(self, mock_k8s_client):
"""With ``egress.disable_ipv6`` false, execd init is not privileged and does not write disable_ipv6."""
provider = BatchSandboxProvider(
mock_k8s_client,
_app_config_with_egress_disable_ipv6(False),
)
mock_k8s_client.create_custom_object.return_value = {
"metadata": {"name": "test-id", "uid": "test-uid"}
}

network_policy = NetworkPolicy(
default_action="deny",
egress=[NetworkRule(action="allow", target="example.com")],
)

provider.create_workload(
sandbox_id="test-id",
namespace="test-ns",
image_spec=ImageSpec(uri="python:3.11"),
entrypoint=["/bin/bash"],
env={},
resource_limits={},
labels={},
expires_at=None,
execd_image="execd:latest",
network_policy=network_policy,
egress_image="opensandbox/egress:v1.0.3",
)

body = mock_k8s_client.create_custom_object.call_args.kwargs["body"]
pod_spec = body["spec"]["template"]["spec"]
execd_init = pod_spec["initContainers"][0]
assert execd_init["name"] == "execd-installer"
assert "securityContext" not in execd_init
assert "/proc/sys/net/ipv6/conf/all/disable_ipv6" not in execd_init["args"][0]

def test_create_workload_with_network_policy_drops_net_admin_from_main_container(self, mock_k8s_client):
"""
Test case: Verify main container drops NET_ADMIN when network_policy is enabled
Expand Down
2 changes: 1 addition & 1 deletion server/tests/k8s/test_egress_helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ def test_handles_missing_default_action(self):
assert "egress" in policy_dict

def test_security_context_adds_net_admin_not_privileged(self):
"""Egress sidecar uses NET_ADMIN only (IPv6 is disabled in execd init when egress is on)."""
"""Egress sidecar uses NET_ADMIN only (IPv6 sysctl is optional via ``egress.disable_ipv6``)."""
egress_image = "opensandbox/egress:v1.0.4"
network_policy = NetworkPolicy(
default_action="deny",
Expand Down
4 changes: 3 additions & 1 deletion server/tests/test_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -692,6 +692,8 @@ def test_kubernetes_runtime_with_firecracker_is_valid():


def test_egress_config_mode_literal():
assert EgressConfig(image="opensandbox/egress:v1").mode == EGRESS_MODE_DNS
base = EgressConfig(image="opensandbox/egress:v1")
assert base.mode == EGRESS_MODE_DNS
assert base.disable_ipv6 is True
cfg = EgressConfig(image="opensandbox/egress:v1", mode=EGRESS_MODE_DNS_NFT)
assert cfg.mode == EGRESS_MODE_DNS_NFT
Loading