Add Vault Agent sidecar to CSI Provider (#749)

Adds Agent as a sidecar for the CSI Provider to: * Cache k8s auth login leases * Cache secret leases * Automatically renew renewable leases in the background
hashicorp · Apr 6, 2023 · 0fe9164 · 0fe9164
1 parent fc7d432
commit 0fe9164
Show file tree

Hide file tree

Showing 10 changed files with 366 additions and 66 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -8,6 +8,7 @@ Features:
 * server: New `extraPorts` option for adding ports to the Vault server statefulset [GH-841](https://github.com/hashicorp/vault-helm/pull/841)
 * server: Add configurable Port Number in readinessProbe and livenessProbe for the server-statefulset [GH-831](https://github.com/hashicorp/vault-helm/pull/831)
 * injector: Make livenessProbe and readinessProbe configurable and add configurable startupProbe [GH-852](https://github.com/hashicorp/vault-helm/pull/852)
+* csi: Add an Agent sidecar to Vault CSI Provider pods to provide lease caching and renewals [GH-749](https://github.com/hashicorp/vault-helm/pull/749)
 
 ## 0.23.0 (November 28th, 2022)
 

diff --git a/templates/_helpers.tpl b/templates/_helpers.tpl
@@ -778,6 +778,16 @@ Sets the container resources if the user has set any.
   {{ end }}
 {{- end -}}
 
+{{/*
+Sets the container resources for CSI's Agent sidecar if the user has set any.
+*/}}
+{{- define "csi.agent.resources" -}}
+  {{- if .Values.csi.agent.resources -}}
+          resources:
+{{ toYaml .Values.csi.agent.resources | indent 12}}
+  {{ end }}
+{{- end -}}
+
 {{/*
 Sets extra CSI daemonset annotations
 */}}

diff --git a/templates/csi-agent-configmap.yaml b/templates/csi-agent-configmap.yaml
@@ -0,0 +1,29 @@
+{{- template "vault.csiEnabled" . -}}
+{{- if and (.csiEnabled) (eq (.Values.csi.agent.enabled | toString) "true") -}}
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ template "vault.fullname" . }}-csi-provider-agent-config
+  namespace: {{ .Release.Namespace }}
+  labels:
+    helm.sh/chart: {{ include "vault.chart" . }}
+    app.kubernetes.io/name: {{ include "vault.name" . }}-csi-provider
+    app.kubernetes.io/instance: {{ .Release.Name }}
+    app.kubernetes.io/managed-by: {{ .Release.Service }}
+data:
+  config.hcl: |
+    vault {
+        {{- if .Values.global.externalVaultAddr }}
+        "address" = "{{ .Values.global.externalVaultAddr }}"
+        {{- else }}
+        "address" = "{{ include "vault.scheme" . }}://{{ template "vault.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.server.service.port }}"
+        {{- end }}
+    }
+
+    cache {}
+
+    listener "unix" {
+        address = "/var/run/vault/agent.sock"
+        tls_disable = true
+    }
+{{- end }}
diff --git a/templates/csi-daemonset.yaml b/templates/csi-daemonset.yaml
@@ -55,21 +55,24 @@ spec:
             - --endpoint=/provider/vault.sock
             - --debug={{ .Values.csi.debug }}
             {{- if .Values.csi.extraArgs }}
-              {{- toYaml .Values.csi.extraArgs | nindent 12 }}
+            {{- toYaml .Values.csi.extraArgs | nindent 12 }}
             {{- end }}
           env:
             - name: VAULT_ADDR
-            {{- if .Values.global.externalVaultAddr }}
+            {{- if eq (.Values.csi.agent.enabled | toString) "true" }}
+              value: "unix:///var/run/vault/agent.sock"
+            {{- else if .Values.global.externalVaultAddr }}
               value: "{{ .Values.global.externalVaultAddr }}"
             {{- else }}
               value: {{ include "vault.scheme" . }}://{{ template "vault.fullname" . }}.{{ .Release.Namespace }}.svc:{{ .Values.server.service.port }}
             {{- end }}
           volumeMounts:
             - name: providervol
               mountPath: "/provider"
-            - name: mountpoint-dir
-              mountPath: {{ .Values.csi.daemonSet.kubeletRootDir }}/pods
-              mountPropagation: HostToContainer
+            {{- if eq (.Values.csi.agent.enabled | toString) "true" }}
+            - name: agent-unix-socket
+              mountPath: /var/run/vault
+            {{- end }}
             {{- if .Values.csi.volumeMounts }}
               {{- toYaml .Values.csi.volumeMounts | nindent 12}}
             {{- end }}
@@ -91,15 +94,57 @@ spec:
             periodSeconds: {{ .Values.csi.readinessProbe.periodSeconds }}
             successThreshold: {{ .Values.csi.readinessProbe.successThreshold }}
             timeoutSeconds: {{ .Values.csi.readinessProbe.timeoutSeconds }}
+        {{- if eq (.Values.csi.agent.enabled | toString) "true" }}
+        - name: {{ include "vault.name" . }}-agent
+          image: "{{ .Values.csi.agent.image.repository }}:{{ .Values.csi.agent.image.tag }}"
+          imagePullPolicy: {{ .Values.csi.agent.image.pullPolicy }}
+          {{ template "csi.agent.resources" . }}
+          command:
+            - vault
+          args:
+            - agent
+            - -config=/etc/vault/config.hcl
+            {{- if .Values.csi.agent.extraArgs }}
+            {{- toYaml .Values.csi.agent.extraArgs | nindent 12 }}
+            {{- end }}
+          ports:
+            - containerPort: 8200
+          env:
+            - name: VAULT_LOG_LEVEL
+              value: "{{ .Values.csi.agent.logLevel }}"
+            - name: VAULT_LOG_FORMAT
+              value: "{{ .Values.csi.agent.logFormat }}"
+          securityContext:
+            runAsNonRoot: true
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            runAsUser: 100
+            runAsGroup: 1000
+          volumeMounts:
+            - name: agent-config
+              mountPath: /etc/vault/config.hcl
+              subPath: config.hcl
+              readOnly: true
+            - name: agent-unix-socket
+              mountPath: /var/run/vault
+            {{- if .Values.csi.volumeMounts }}
+            {{- toYaml .Values.csi.volumeMounts | nindent 12 }}
+            {{- end }}
+        {{- end }}
       volumes:
         - name: providervol
           hostPath:
             path: {{ .Values.csi.daemonSet.providersDir }}
-        - name: mountpoint-dir
-          hostPath:
-            path: {{ .Values.csi.daemonSet.kubeletRootDir }}/pods
-       {{- if .Values.csi.volumes }}
-         {{- toYaml .Values.csi.volumes | nindent 8}}
-       {{- end }}
+        {{- if eq (.Values.csi.agent.enabled | toString) "true" }}
+        - name: agent-config
+          configMap:
+            name: {{ template "vault.fullname" . }}-csi-provider-agent-config
+        - name: agent-unix-socket
+          emptyDir:
+            medium: Memory
+        {{- end }}
+        {{- if .Values.csi.volumes }}
+        {{- toYaml .Values.csi.volumes | nindent 8}}
+        {{- end }}
       {{- include "imagePullSecrets" . | nindent 6 }}
 {{- end }}
diff --git a/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml b/test/acceptance/csi-test/vault-kv-secretproviderclass.yaml
@@ -2,15 +2,14 @@
 # SPDX-License-Identifier: MPL-2.0
 
 # The "Hello World" Vault SecretProviderClass
-apiVersion: secrets-store.csi.x-k8s.io/v1alpha1
+apiVersion: secrets-store.csi.x-k8s.io/v1
 kind: SecretProviderClass
 metadata:
   name: vault-kv
 spec:
   provider: vault
   parameters:
     roleName: "kv-role"
-    vaultAddress: http://vault:8200
     objects: |
       - objectName: "bar"
         secretPath: "secret/data/kv1"

diff --git a/test/acceptance/csi.bats b/test/acceptance/csi.bats
@@ -9,30 +9,36 @@ load _helpers
   kubectl create namespace acceptance
 
   # Install Secrets Store CSI driver
-  CSI_DRIVER_VERSION=1.0.0
-  helm install secrets-store-csi-driver https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts/secrets-store-csi-driver-${CSI_DRIVER_VERSION}.tgz?raw=true \
+  # Configure it to pass in a JWT for the provider to use, and rotate secrets rapidly
+  # so we can see Agent's cache working.
+  CSI_DRIVER_VERSION=1.3.2
+  helm install secrets-store-csi-driver secrets-store-csi-driver \
+    --repo https://kubernetes-sigs.github.io/secrets-store-csi-driver/charts \
+    --version=$(CSI_DRIVER_VERSION) \
     --wait --timeout=5m \
     --namespace=acceptance \
     --set linux.image.pullPolicy="IfNotPresent" \
-    --set syncSecret.enabled=true
+    --set tokenRequests[0].audience="vault" \
+    --set enableSecretRotation=true \
+    --set rotationPollInterval=5s
   # Install Vault and Vault provider
   helm install vault \
     --wait --timeout=5m \
     --namespace=acceptance \
     --set="server.dev.enabled=true" \
     --set="csi.enabled=true" \
-    --set="injector.enabled=false" .
+    --set="csi.debug=true" \
+    --set="csi.agent.logLevel=debug" \
+    --set="injector.enabled=false" \
+    .
   kubectl --namespace=acceptance wait --for=condition=Ready --timeout=5m pod -l app.kubernetes.io/name=vault
   kubectl --namespace=acceptance wait --for=condition=Ready --timeout=5m pod -l app.kubernetes.io/name=vault-csi-provider
 
   # Set up k8s auth and a kv secret.
   cat ./test/acceptance/csi-test/vault-policy.hcl | kubectl --namespace=acceptance exec -i vault-0 -- vault policy write kv-policy -
   kubectl --namespace=acceptance exec vault-0 -- vault auth enable kubernetes
   kubectl --namespace=acceptance exec vault-0 -- sh -c 'vault write auth/kubernetes/config \
-    token_reviewer_jwt="$(cat /var/run/secrets/kubernetes.io/serviceaccount/token)" \
-    kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443" \
-    kubernetes_ca_cert=@/var/run/secrets/kubernetes.io/serviceaccount/ca.crt \
-    disable_iss_validation=true'
+    kubernetes_host="https://$KUBERNETES_PORT_443_TCP_ADDR:443"'
   kubectl --namespace=acceptance exec vault-0 -- vault write auth/kubernetes/role/kv-role \
     bound_service_account_names=nginx \
     bound_service_account_namespaces=acceptance \
@@ -46,6 +52,22 @@ load _helpers
 
   result=$(kubectl --namespace=acceptance exec nginx -- cat /mnt/secrets-store/bar)
   [[ "$result" == "hello1" ]]
+
+  for i in $(seq 10); do
+    sleep 2
+    if [ "$(kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-agent | grep "returning cached response: path=/v1/auth/kubernetes/login")" ]; then
+        echo "Agent returned a cached login response"
+        return
+    fi
+
+    echo "Waiting for a cached response from Agent..."
+  done
+
+  # Print the logs and fail the test
+  echo "Failed to find a log for a cached Agent response"
+  kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-agent
+  kubectl --namespace=acceptance logs --tail=-1 -l "app.kubernetes.io/name=vault-csi-provider" -c vault-csi-provider
+  exit 1
 }
 
 # Clean up

diff --git a/test/unit/csi-agent-configmap.bats b/test/unit/csi-agent-configmap.bats
@@ -0,0 +1,45 @@
+#!/usr/bin/env bats
+
+load _helpers
+
+@test "csi/Agent-ConfigMap: disabled by default" {
+  cd `chart_dir`
+  local actual=$( (helm template \
+      --show-only templates/csi-agent-configmap.yaml  \
+      . || echo "---") | tee /dev/stderr |
+      yq 'length > 0' | tee /dev/stderr)
+  [ "${actual}" = "false" ]
+}
+
+@test "csi/Agent-ConfigMap: name" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/csi-agent-configmap.yaml \
+      --set "csi.enabled=true" \
+      . | tee /dev/stderr |
+      yq -r '.metadata.name' | tee /dev/stderr)
+  [ "${actual}" = "release-name-vault-csi-provider-agent-config" ]
+}
+
+@test "csi/Agent-ConfigMap: Vault addr not affected by injector setting" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/csi-agent-configmap.yaml \
+      --set "csi.enabled=true" \
+      --release-name not-external-test \
+      --set 'injector.externalVaultAddr=http://vault-outside' \
+      . | tee /dev/stderr |
+      yq -r '.data["config.hcl"]' | tee /dev/stderr)
+  echo "${actual}" | grep "http://not-external-test-vault.default.svc:8200"
+}
+
+@test "csi/Agent-ConfigMap: Vault addr correctly set for externalVaultAddr" {
+  cd `chart_dir`
+  local actual=$(helm template \
+      --show-only templates/csi-agent-configmap.yaml \
+      --set "csi.enabled=true" \
+      --set 'global.externalVaultAddr=http://vault-outside' \
+      . | tee /dev/stderr |
+      yq -r '.data["config.hcl"]' | tee /dev/stderr)
+  echo "${actual}" | grep "http://vault-outside"
+}