From f25b44eae88456597fd0f460145d3ec14041604d Mon Sep 17 00:00:00 2001 From: joycej Date: Fri, 17 Feb 2023 12:34:54 -0800 Subject: [PATCH 1/8] Ztunnel manifests --- internal/assets/manifests/ztunnel/Chart.yaml | 14 + internal/assets/manifests/ztunnel/README.md | 36 +++ .../manifests/ztunnel/templates/NOTES.txt | 5 + .../manifests/ztunnel/templates/_helpers.tpl | 41 +++ .../ztunnel/templates/daemonset.yaml | 281 ++++++++++++++++++ .../manifests/ztunnel/templates/rbac.yaml | 16 + internal/assets/manifests/ztunnel/values.yaml | 52 ++++ 7 files changed, 445 insertions(+) create mode 100644 internal/assets/manifests/ztunnel/Chart.yaml create mode 100644 internal/assets/manifests/ztunnel/README.md create mode 100644 internal/assets/manifests/ztunnel/templates/NOTES.txt create mode 100644 internal/assets/manifests/ztunnel/templates/_helpers.tpl create mode 100644 internal/assets/manifests/ztunnel/templates/daemonset.yaml create mode 100644 internal/assets/manifests/ztunnel/templates/rbac.yaml create mode 100644 internal/assets/manifests/ztunnel/values.yaml diff --git a/internal/assets/manifests/ztunnel/Chart.yaml b/internal/assets/manifests/ztunnel/Chart.yaml new file mode 100644 index 000000000..a315bfe34 --- /dev/null +++ b/internal/assets/manifests/ztunnel/Chart.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +name: ztunnel +# This version is never actually shipped. istio/release-builder will replace it at build-time +# with the appropriate version +version: 1.0.0 +appVersion: 1.0.0 +description: Helm chart for istio ztunnel components +keywords: + - istio-ztunnel + - istio +sources: + - http://github.com/istio/istio +engine: gotpl +icon: https://istio.io/latest/favicons/android-192x192.png diff --git a/internal/assets/manifests/ztunnel/README.md b/internal/assets/manifests/ztunnel/README.md new file mode 100644 index 000000000..f8887a126 --- /dev/null +++ b/internal/assets/manifests/ztunnel/README.md @@ -0,0 +1,36 @@ +# Istio Ztunnel Helm Chart + +This chart installs an Istio ztunnel. + +## Setup Repo Info + +```console +helm repo add istio https://istio-release.storage.googleapis.com/charts +helm repo update +``` + +_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._ + +## Installing the Chart + +To install the chart: + +```console +helm install ztunnel istio/ztunnel +``` + +## Uninstalling the Chart + +To uninstall/delete the chart: + +```console +helm delete ztunnel +``` + +## Configuration + +To view support configuration options and documentation, run: + +```console +helm show values istio/ztunnel +``` diff --git a/internal/assets/manifests/ztunnel/templates/NOTES.txt b/internal/assets/manifests/ztunnel/templates/NOTES.txt new file mode 100644 index 000000000..93297520e --- /dev/null +++ b/internal/assets/manifests/ztunnel/templates/NOTES.txt @@ -0,0 +1,5 @@ +ztunnel successfully installed! + +To learn more about the release, try: + $ helm status {{ .Release.Name }} + $ helm get all {{ .Release.Name }} diff --git a/internal/assets/manifests/ztunnel/templates/_helpers.tpl b/internal/assets/manifests/ztunnel/templates/_helpers.tpl new file mode 100644 index 000000000..a57569b78 --- /dev/null +++ b/internal/assets/manifests/ztunnel/templates/_helpers.tpl @@ -0,0 +1,41 @@ +{{- define "revision" -}} +{{- default "default" (.Values.revision | replace "." "-") -}} +{{- end -}} + +{{- define "namespaced-revision" -}} +{{- $revision := (include "revision" .) -}} +{{- if eq $revision "default" -}} +{{- printf "%s" $revision -}} +{{- else -}} +{{- printf "%s.%s" $revision .Release.Namespace -}} +{{- end -}} +{{- end -}} + +{{- define "name-with-revision" -}} +{{- if .context.Values.revision -}} +{{- printf "%s-%s" .name (include "revision" .context) -}} +{{- else -}} +{{- .name -}} +{{- end -}} +{{- end -}} + +{{- define "name-with-namespaced-revision" -}} +{{- if .context.Values.revision -}} +{{- printf "%s-%s" (include "name-with-revision" .) .context.Release.Namespace -}} +{{- else -}} +{{- .name -}} +{{- end -}} +{{- end -}} + +{{- define "toYamlIf" }} +{{- if .value }} +{{- if .key }} +{{ .key }}: +{{- end }} +{{- if gt (.indent | int) 0 }} +{{ .value | toYaml | indent .indent }} +{{- else }} +{{ .value | toYaml }} +{{- end }} +{{- end }} +{{- end }} diff --git a/internal/assets/manifests/ztunnel/templates/daemonset.yaml b/internal/assets/manifests/ztunnel/templates/daemonset.yaml new file mode 100644 index 000000000..2ef321655 --- /dev/null +++ b/internal/assets/manifests/ztunnel/templates/daemonset.yaml @@ -0,0 +1,281 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: ztunnel + namespace: {{ .Release.Namespace }} + labels: + {{- .Values.labels | toYaml | nindent 4}} + annotations: + {{- .Values.annotations | toYaml | nindent 4 }} +spec: + updateStrategy: + rollingUpdate: + maxSurge: 1 + maxUnavailable: 0 + selector: + matchLabels: + app: ztunnel + template: + metadata: + labels: + sidecar.istio.io/inject: "false" + app: ztunnel +{{ with .Values.podLabels -}}{{ toYaml . | indent 8 }}{{ end }} + annotations: + ambient.istio.io/redirection: disabled + sidecar.istio.io/inject: "false" +{{ with .Values.podAnnotations -}}{{ toYaml . | indent 8 }}{{ end }} + spec: + serviceAccountName: ztunnel + tolerations: + - effect: NoSchedule + operator: Exists + - key: CriticalAddonsOnly + operator: Exists + - effect: NoExecute + operator: Exists + initContainers: + - name: istio-init +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + {{/* TODO(https://github.com/istio/istio/issues/43243): use distroless, but we are depending on a variant of things not in the distroless image */}} + image: "{{ .Values.hub }}/{{ .Values.image | default "ztunnel" }}:{{ .Values.tag }}{{with (.Values.variant )}}-{{.}}{{end}}" +{{- end }} +{{- with .Values.imagePullPolicy }} + imagePullPolicy: {{ . }} +{{- end }} + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + command: + - sh + - -c + - | + set -ex + + PROXY_ORG_SRC_MARK=0x4d2/0xfff + # tproxy mark, it's only used here. + MARK=0x400/0xfff + ORG_SRC_RET_MARK=0x4d3/0xfff + + # Below is from config.sh but used in redirect-worker.sh as well + POD_OUTBOUND=15001 + POD_INBOUND=15008 + POD_INBOUND_PLAINTEXT=15006 + + # socket mark setup + OUTBOUND_MASK="0x100" + OUTBOUND_MARK="0x100/$OUTBOUND_MASK" + + SKIP_MASK="0x200" + SKIP_MARK="0x200/$SKIP_MASK" + + # note!! this includes the skip mark bit, so match on skip mark will match this as well. + CONNSKIP_MASK="0x220" + CONNSKIP_MARK="0x220/$CONNSKIP_MASK" + + # note!! this includes the skip mark bit, so match on skip mark will match this as well. + PROXY_MASK="0x210" + PROXY_MARK="0x210/$PROXY_MASK" + + PROXY_RET_MASK="0x040" + PROXY_RET_MARK="0x040/$PROXY_RET_MASK" + + INBOUND_TUN=istioin + OUTBOUND_TUN=istioout + + # TODO: look into why link local (169.254.x.x) address didn't work + # they don't respond to ARP. + INBOUND_TUN_IP=192.168.126.1 + ZTUNNEL_INBOUND_TUN_IP=192.168.126.2 + OUTBOUND_TUN_IP=192.168.127.1 + ZTUNNEL_OUTBOUND_TUN_IP=192.168.127.2 + TUN_PREFIX=30 + + # a route table number number we can use to send traffic to envoy (should be unused). + INBOUND_ROUTE_TABLE=100 + INBOUND_ROUTE_TABLE2=103 + OUTBOUND_ROUTE_TABLE=101 + # needed for original src. + PROXY_ROUTE_TABLE=102 + + set +e # Only for delete, we don't care if this fails + ip link del p$INBOUND_TUN + ip link del p$OUTBOUND_TUN + set -e +{{- if not (.Capabilities.KubeVersion.GitVersion | contains "-eks") }} + HOST_IP=$(ip route | grep default | awk '{print $3}') +{{- end }} + + ip link add name p$INBOUND_TUN type geneve id 1000 remote $HOST_IP + ip addr add $ZTUNNEL_INBOUND_TUN_IP/$TUN_PREFIX dev p$INBOUND_TUN + + ip link add name p$OUTBOUND_TUN type geneve id 1001 remote $HOST_IP + ip addr add $ZTUNNEL_OUTBOUND_TUN_IP/$TUN_PREFIX dev p$OUTBOUND_TUN + + ip link set p$INBOUND_TUN up + ip link set p$OUTBOUND_TUN up + + echo 0 > /proc/sys/net/ipv4/conf/p$INBOUND_TUN/rp_filter + echo 0 > /proc/sys/net/ipv4/conf/p$OUTBOUND_TUN/rp_filter + + set +e # Only for delete, we don't care if this fails + ip rule del priority 20000 + ip rule del priority 20001 + ip rule del priority 20002 + ip rule del priority 20003 + + ip route flush table 100 + ip route flush table 101 + ip route flush table 102 + set -e + + ip rule add priority 20000 fwmark $MARK lookup 100 + ip rule add priority 20003 fwmark $ORG_SRC_RET_MARK lookup 100 + ip route add local 0.0.0.0/0 dev lo table 100 + + ip route add table 101 $HOST_IP dev eth0 scope link + ip route add table 101 0.0.0.0/0 via $OUTBOUND_TUN_IP dev p$OUTBOUND_TUN + + ip route add table 102 $HOST_IP dev eth0 scope link + ip route add table 102 0.0.0.0/0 via $INBOUND_TUN_IP dev p$INBOUND_TUN + + set +e + num_legacy_lines=$( (iptables-legacy-save || true; ip6tables-legacy-save || true) 2>/dev/null | grep '^-' | wc -l) + if [ "${num_legacy_lines}" -ge 10 ]; then + mode=legacy + else + num_nft_lines=$( (timeout 5 sh -c "iptables-nft-save; ip6tables-nft-save" || true) 2>/dev/null | grep '^-' | wc -l) + if [ "${num_legacy_lines}" -ge "${num_nft_lines}" ]; then + mode=legacy + else + mode=nft + fi + fi + IPTABLES=iptables-legacy + if [ "${mode}" = "nft" ]; then + IPTABLES=iptables-nft + fi + set -e + + $IPTABLES -t mangle -F PREROUTING + $IPTABLES -t nat -F OUTPUT + + $IPTABLES -t mangle -A PREROUTING -p tcp -i p$INBOUND_TUN -m tcp --dport=$POD_INBOUND -j TPROXY --tproxy-mark $MARK --on-port $POD_INBOUND --on-ip 127.0.0.1 + $IPTABLES -t mangle -A PREROUTING -p tcp -i p$OUTBOUND_TUN -j TPROXY --tproxy-mark $MARK --on-port $POD_OUTBOUND --on-ip 127.0.0.1 + $IPTABLES -t mangle -A PREROUTING -p tcp -i p$INBOUND_TUN -j TPROXY --tproxy-mark $MARK --on-port $POD_INBOUND_PLAINTEXT --on-ip 127.0.0.1 + + $IPTABLES -t mangle -A PREROUTING -p tcp -i eth0 ! --dst $INSTANCE_IP -j MARK --set-mark $ORG_SRC_RET_MARK + # With normal linux routing we need to disable the rp_filter + # as we get packets from a tunnel that doesn't have default routes. + echo 0 > /proc/sys/net/ipv4/conf/all/rp_filter + echo 0 > /proc/sys/net/ipv4/conf/default/rp_filter + echo 0 > /proc/sys/net/ipv4/conf/eth0/rp_filter + env: + - name: INSTANCE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: HOST_IP + valueFrom: + fieldRef: + fieldPath: status.hostIP + {{- if .Values.meshConfig.defaultConfig.proxyMetadata }} + {{- range $key, $value := .Values.meshConfig.defaultConfig.proxyMetadata}} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + {{- end }} + {{- with .Values.env }} + {{- range $key, $val := . }} + - name: {{ $key }} + value: "{{ $val }}" + {{- end }} + {{- end }} + containers: + - name: istio-proxy +{{- if contains "/" .Values.image }} + image: "{{ .Values.image }}" +{{- else }} + image: "{{ .Values.hub }}/{{ .Values.image | default "ztunnel" }}:{{ .Values.tag }}{{with (.Values.variant )}}-{{.}}{{end}}" +{{- end }} +{{- with .Values.imagePullPolicy }} + imagePullPolicy: {{ . }} +{{- end }} + securityContext: + allowPrivilegeEscalation: false + privileged: false + capabilities: + drop: + - ALL + add: + - NET_ADMIN + readOnlyRootFilesystem: true + runAsGroup: 1337 + runAsNonRoot: false + runAsUser: 0 + readinessProbe: + httpGet: + port: 15021 + path: /healthz/ready + args: + - proxy + - ztunnel + env: + - name: CLUSTER_ID + value: {{ .Values.multiCluster.clusterName | default "Kubernetes" }} + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: INSTANCE_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SERVICE_ACCOUNT + valueFrom: + fieldRef: + fieldPath: spec.serviceAccountName + {{- if .Values.meshConfig.defaultConfig.proxyMetadata }} + {{- range $key, $value := .Values.meshConfig.defaultConfig.proxyMetadata}} + - name: {{ $key }} + value: "{{ $value }}" + {{- end }} + {{- end }} + {{- with .Values.env }} + {{- range $key, $val := . }} + - name: {{ $key }} + value: "{{ $val }}" + {{- end }} + {{- end }} + volumeMounts: + - mountPath: /var/run/secrets/istio + name: istiod-ca-cert + - mountPath: /var/run/secrets/tokens + name: istio-token + volumes: + - name: istio-token + projected: + sources: + - serviceAccountToken: + path: istio-token + expirationSeconds: 43200 + audience: istio-ca + - name: istiod-ca-cert + configMap: + name: istio-ca-root-cert \ No newline at end of file diff --git a/internal/assets/manifests/ztunnel/templates/rbac.yaml b/internal/assets/manifests/ztunnel/templates/rbac.yaml new file mode 100644 index 000000000..5a569b647 --- /dev/null +++ b/internal/assets/manifests/ztunnel/templates/rbac.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: ServiceAccount + {{- with .Values.imagePullSecrets }} +imagePullSecrets: + {{- range . }} + - name: {{ . }} + {{- end }} + {{- end }} +metadata: + name: ztunnel + namespace: {{ .Release.Namespace }} + labels: + {{- .Values.labels | toYaml | nindent 4}} + annotations: + {{- .Values.annotations | toYaml | nindent 4 }} +--- diff --git a/internal/assets/manifests/ztunnel/values.yaml b/internal/assets/manifests/ztunnel/values.yaml new file mode 100644 index 000000000..b184c718f --- /dev/null +++ b/internal/assets/manifests/ztunnel/values.yaml @@ -0,0 +1,52 @@ +# Hub to pull from. Image will be `Hub/Image:Tag-Variant` +hub: "gcr.io/istio-testing" +# Tag to pull from. Image will be `Hub/Image:Tag-Variant` +tag: "latest" +# Variant to pull. Options are "debug" or "distroless". Unset will use the default for the given version. +variant: "" + +# Image name to pull from. Image will be `Hub/Image:Tag-Variant` +# If Image contains a "/", it will replace the entire `image` in the pod. +image: ztunnel + +# Labels to apply to all top level resources +labels: {} +# Annotations to apply to all top level resources +annotations: {} + +# Annotations added to each pod. The default annotations are required for scraping prometheus (in most environments). +podAnnotations: + prometheus.io/port: "15020" + prometheus.io/scrape: "true" + +# Additional labels to apply on the pod level +podLabels: {} + +# Pod resource configuration +resources: + requests: + cpu: 500m + memory: 2048Mi + +# List of secret names to add to the service account as image pull secrets +imagePullSecrets: [] + +# A `key: value` mapping of environment variables to add to the pod +env: {} + +# Override for the pod imagePullPolicy +imagePullPolicy: "" + +# Settings for multicluster +multiCluster: + # The name of the cluster we are installing in. Note this is a user-defined name, which must be consistent + # with Istiod configuration. + clusterName: "" + +# meshConfig defines runtime configuration of components. +# For ztunnel, only defaultConfig is used, but this is nested under `meshConfig` for consistency with other +# components. +# TODO: https://github.com/istio/istio/issues/43248 +meshConfig: + defaultConfig: + proxyMetadata: {} From 327c9852cb8a99f25f4ea860c9f9d6e1f8d9ef78 Mon Sep 17 00:00:00 2001 From: joycej Date: Tue, 21 Feb 2023 12:38:53 -0800 Subject: [PATCH 2/8] Changes related to ambient Includes: Assest change, add ambientTopology to CRD a reconciler for the ztunnel --- Makefile | 6 +- api/v1alpha1/istiocontrolplane.gen.json | 13 + api/v1alpha1/istiocontrolplane.pb.go | 1122 +++++++++-------- api/v1alpha1/istiocontrolplane.pb.html | 37 +- api/v1alpha1/istiocontrolplane.proto | 6 + .../istiocontrolplane_deepcopy.gen.go | 21 + api/v1alpha1/istiocontrolplane_json.gen.go | 11 + config/crd/bases/istio-operator-crds.gen.yaml | 12 + config/manager/kustomization.yaml | 1 + controllers/istiocontrolplane_controller.go | 8 + .../crds/istio-operator-crds.gen.yaml | 12 + go.mod | 2 +- internal/assets/assets.go | 5 + .../assets/manifests/ztunnel/values.yaml.tpl | 52 + internal/components/ztunnel/reconcile.go | 104 ++ 15 files changed, 888 insertions(+), 524 deletions(-) create mode 100644 internal/assets/manifests/ztunnel/values.yaml.tpl create mode 100644 internal/components/ztunnel/reconcile.go diff --git a/Makefile b/Makefile index 4884ebe4b..0475470d1 100644 --- a/Makefile +++ b/Makefile @@ -23,7 +23,8 @@ KUSTOMIZE_VERSION = 4.1.2 ISTIO_VERSION = 1.16.1 BUF_VERSION = 1.7.0 -PATH := $(PATH):$(PWD)/bin +#PATH := $(PATH):$(PWD)/bin +PATH := $(PWD)/bin:$(PATH) all: check manager @@ -70,8 +71,9 @@ ifneq (${SKIP_TESTS}, 1) endif # Build manager binary +#manager: generate manifests fmt vet build .PHONY: manager -manager: generate manifests fmt vet build +manager: generate manifests fmt build # Build manager binary .PHONY: build diff --git a/api/v1alpha1/istiocontrolplane.gen.json b/api/v1alpha1/istiocontrolplane.gen.json index d556f5cdc..ec1d7680a 100644 --- a/api/v1alpha1/istiocontrolplane.gen.json +++ b/api/v1alpha1/istiocontrolplane.gen.json @@ -1886,6 +1886,16 @@ } } }, + "istio_operator.v2.api.v1alpha1.AmbientTopology": { + "description": "Ambient sets Istiod in an ambient topology", + "type": "object", + "properties": { + "enabled": { + "type": "boolean", + "nullable": true + } + } + }, "istio_operator.v2.api.v1alpha1.BaseKubernetesContainerConfiguration": { "type": "object", "properties": { @@ -2312,6 +2322,9 @@ }, "sidecarInjector": { "$ref": "#/components/schemas/istio_operator.v2.api.v1alpha1.SidecarInjectorConfiguration" + }, + "ambientTopology": { + "$ref": "#/components/schemas/istio_operator.v2.api.v1alpha1.AmbientTopology" } } }, diff --git a/api/v1alpha1/istiocontrolplane.pb.go b/api/v1alpha1/istiocontrolplane.pb.go index a17b877f3..b9eb78662 100644 --- a/api/v1alpha1/istiocontrolplane.pb.go +++ b/api/v1alpha1/istiocontrolplane.pb.go @@ -350,6 +350,7 @@ type IstioControlPlaneSpec struct { NetworkName string `protobuf:"bytes,23,opt,name=networkName,proto3" json:"networkName,omitempty"` // Standalone sidecar injector configuration. SidecarInjector *SidecarInjectorConfiguration `protobuf:"bytes,24,opt,name=sidecarInjector,proto3" json:"sidecarInjector,omitempty"` + AmbientTopology *AmbientTopology `protobuf:"bytes,25,opt,name=ambientTopology,proto3" json:"ambientTopology,omitempty"` } func (x *IstioControlPlaneSpec) Reset() { @@ -552,6 +553,13 @@ func (x *IstioControlPlaneSpec) GetSidecarInjector() *SidecarInjectorConfigurati return nil } +func (x *IstioControlPlaneSpec) GetAmbientTopology() *AmbientTopology { + if x != nil { + return x.AmbientTopology + } + return nil +} + type SidecarInjectorConfiguration struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -880,7 +888,8 @@ type SDSConfiguration struct { // The JWT token for SDS and the aud field of such JWT. See RFC 7519, section 4.1.3. // When a CSR is sent from Citadel Agent to the CA (e.g. Citadel), this aud is to make sure the - // JWT is intended for the CA. + // + // JWT is intended for the CA. TokenAudience string `protobuf:"bytes,1,opt,name=tokenAudience,proto3" json:"tokenAudience,omitempty"` } @@ -1763,6 +1772,54 @@ func (x *HTTPProxyEnvsConfiguration) GetNoProxy() string { return "" } +// Ambient sets Istiod in an ambient topology +type AmbientTopology struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Enabled *wrappers.BoolValue `protobuf:"bytes,1,opt,name=enabled,proto3" json:"enabled,omitempty"` +} + +func (x *AmbientTopology) Reset() { + *x = AmbientTopology{} + if protoimpl.UnsafeEnabled { + mi := &file_api_v1alpha1_istiocontrolplane_proto_msgTypes[18] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *AmbientTopology) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AmbientTopology) ProtoMessage() {} + +func (x *AmbientTopology) ProtoReflect() protoreflect.Message { + mi := &file_api_v1alpha1_istiocontrolplane_proto_msgTypes[18] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AmbientTopology.ProtoReflect.Descriptor instead. +func (*AmbientTopology) Descriptor() ([]byte, []int) { + return file_api_v1alpha1_istiocontrolplane_proto_rawDescGZIP(), []int{18} +} + +func (x *AmbientTopology) GetEnabled() *wrappers.BoolValue { + if x != nil { + return x.Enabled + } + return nil +} + //