diff --git a/.github/workflows/ci-build.yaml b/.github/workflows/ci-build.yaml index a833f7fc9..98fcde29d 100644 --- a/.github/workflows/ci-build.yaml +++ b/.github/workflows/ci-build.yaml @@ -48,8 +48,9 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - k3s-version: [ v1.27.1 ] - # k3s-version: [v1.20.2, v1.19.2, v1.18.9, v1.17.11, v1.16.15] + k3s-version: + - v1.27.1-k3s1 + - v1.33.5-k3s1 steps: - name: Download kuttl plugin env: @@ -69,7 +70,20 @@ jobs: set -x curl -s https://raw.githubusercontent.com/rancher/k3d/main/install.sh | bash sudo mkdir -p $HOME/.kube && sudo chown -R runner $HOME/.kube - k3d cluster create --servers 3 --image rancher/k3s:${{ matrix.k3s-version }}-k3s1 + + feature_flags=() + case "${{ matrix.k3s-version }}" in + v1.3[3456789]*) + # Enable ClusterTrustBundle and ClusterTrustBundleProjection until it is enabled by default in kubernetes + feature_flags+=( + "--k3s-arg" "--kube-apiserver-arg=feature-gates=ClusterTrustBundle=true,ClusterTrustBundleProjection=true@server:*" + "--k3s-arg" "--kube-apiserver-arg=runtime-config=certificates.k8s.io/v1beta1/clustertrustbundles=true@server:*" + "--k3s-arg" "--kubelet-arg=feature-gates=ClusterTrustBundle=true,ClusterTrustBundleProjection=true@agent:*" + ) + ;; + esac + + k3d cluster create --servers 3 --image "rancher/k3s:${{ matrix.k3s-version }}" "${feature_flags[@]}" kubectl version k3d version - name: Checkout code diff --git a/api/v1beta1/argocd_types.go b/api/v1beta1/argocd_types.go index e46a1ba91..223349108 100644 --- a/api/v1beta1/argocd_types.go +++ b/api/v1beta1/argocd_types.go @@ -588,6 +588,9 @@ type ArgoCDRepoSpec struct { // Custom labels to pods deployed by the operator Labels map[string]string `json:"labels,omitempty"` + + // Custom certificates to inject into the repo server container and its plugins to trust source hosting sites + SystemCATrust *ArgoCDSystemCATrustSpec `json:"systemCATrust,omitempty"` } func (a *ArgoCDRepoSpec) IsEnabled() bool { @@ -598,6 +601,18 @@ func (a *ArgoCDRepoSpec) IsRemote() bool { return a.Remote != nil && *a.Remote != "" } +// ArgoCDSystemCATrustSpec defines custom certificates to inject into the repo server container and its plugins to trust source hosting sites +type ArgoCDSystemCATrustSpec struct { + // DropImageCertificates will remove all certs that are present in the image, leaving only those explicitly configured here. + DropImageCertificates bool `json:"dropImageCertificates,omitempty"` + // ClusterTrustBundles is a list of projected ClusterTrustBundle volume definitions from where to take the trust certs. + ClusterTrustBundles []corev1.ClusterTrustBundleProjection `json:"clusterTrustBundles,omitempty"` + // Secrets is a list of projected Secret volume definitions from where to take the trust certs. + Secrets []corev1.SecretProjection `json:"secrets,omitempty"` + // ConfigMaps is a list of projected ConfigMap volume definitions from where to take the trust certs. + ConfigMaps []corev1.ConfigMapProjection `json:"configMaps,omitempty"` +} + // ArgoCDRouteSpec defines the desired state for an OpenShift Route. type ArgoCDRouteSpec struct { // Annotations is the map of annotations to use for the Route resource. diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index e2149d612..5de67146a 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -784,6 +784,11 @@ func (in *ArgoCDRepoSpec) DeepCopyInto(out *ArgoCDRepoSpec) { (*out)[key] = val } } + if in.SystemCATrust != nil { + in, out := &in.SystemCATrust, &out.SystemCATrust + *out = new(ArgoCDSystemCATrustSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDRepoSpec. @@ -1136,6 +1141,42 @@ func (in *ArgoCDStatus) DeepCopy() *ArgoCDStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ArgoCDSystemCATrustSpec) DeepCopyInto(out *ArgoCDSystemCATrustSpec) { + *out = *in + if in.ClusterTrustBundles != nil { + in, out := &in.ClusterTrustBundles, &out.ClusterTrustBundles + *out = make([]v1.ClusterTrustBundleProjection, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.Secrets != nil { + in, out := &in.Secrets, &out.Secrets + *out = make([]v1.SecretProjection, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } + if in.ConfigMaps != nil { + in, out := &in.ConfigMaps, &out.ConfigMaps + *out = make([]v1.ConfigMapProjection, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ArgoCDSystemCATrustSpec. +func (in *ArgoCDSystemCATrustSpec) DeepCopy() *ArgoCDSystemCATrustSpec { + if in == nil { + return nil + } + out := new(ArgoCDSystemCATrustSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ArgoCDTLSSpec) DeepCopyInto(out *ArgoCDTLSSpec) { *out = *in diff --git a/bundle/manifests/argoproj.io_argocds.yaml b/bundle/manifests/argoproj.io_argocds.yaml index adc676874..f0dd40e74 100644 --- a/bundle/manifests/argoproj.io_argocds.yaml +++ b/bundle/manifests/argoproj.io_argocds.yaml @@ -20098,6 +20098,237 @@ spec: - name type: object type: array + systemCATrust: + description: Custom certificates to inject into the repo server + container and its plugins to trust source hosting sites + properties: + clusterTrustBundles: + description: ClusterTrustBundles is a list of projected ClusterTrustBundle + volume definitions from where to take the trust certs. + items: + description: |- + ClusterTrustBundleProjection describes how to select a set of + ClusterTrustBundle objects and project their contents into the pod + filesystem. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write + the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + type: array + configMaps: + description: ConfigMaps is a list of projected ConfigMap volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a ConfigMap into a projected volume. + + The contents of the target ConfigMap's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names, + unless the items element is populated with specific mappings of keys to paths. + Note that this is identical to a configmap volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + dropImageCertificates: + description: DropImageCertificates will remove all certs that + are present in the image, leaving only those explicitly + configured here. + type: boolean + secrets: + description: Secrets is a list of projected Secret volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a secret into a projected volume. + + The contents of the target Secret's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names. + Note that this is identical to a secret volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + type: object verifytls: description: VerifyTLS defines whether repo server API should be accessed using strict TLS validation diff --git a/config/crd/bases/argoproj.io_argocds.yaml b/config/crd/bases/argoproj.io_argocds.yaml index c89f73a5d..9d1450312 100644 --- a/config/crd/bases/argoproj.io_argocds.yaml +++ b/config/crd/bases/argoproj.io_argocds.yaml @@ -20087,6 +20087,237 @@ spec: - name type: object type: array + systemCATrust: + description: Custom certificates to inject into the repo server + container and its plugins to trust source hosting sites + properties: + clusterTrustBundles: + description: ClusterTrustBundles is a list of projected ClusterTrustBundle + volume definitions from where to take the trust certs. + items: + description: |- + ClusterTrustBundleProjection describes how to select a set of + ClusterTrustBundle objects and project their contents into the pod + filesystem. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write + the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + type: array + configMaps: + description: ConfigMaps is a list of projected ConfigMap volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a ConfigMap into a projected volume. + + The contents of the target ConfigMap's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names, + unless the items element is populated with specific mappings of keys to paths. + Note that this is identical to a configmap volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + dropImageCertificates: + description: DropImageCertificates will remove all certs that + are present in the image, leaving only those explicitly + configured here. + type: boolean + secrets: + description: Secrets is a list of projected Secret volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a secret into a projected volume. + + The contents of the target Secret's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names. + Note that this is identical to a secret volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + type: object verifytls: description: VerifyTLS defines whether repo server API should be accessed using strict TLS validation diff --git a/controllers/argocd/argocd_controller.go b/controllers/argocd/argocd_controller.go index 44a258846..708c5dc93 100644 --- a/controllers/argocd/argocd_controller.go +++ b/controllers/argocd/argocd_controller.go @@ -351,6 +351,6 @@ func (r *ReconcileArgoCD) internalReconcile(ctx context.Context, request ctrl.Re // SetupWithManager sets up the controller with the Manager. func (r *ReconcileArgoCD) SetupWithManager(mgr ctrl.Manager) error { bldr := ctrl.NewControllerManagedBy(mgr) - r.setResourceWatches(bldr, r.clusterResourceMapper, r.tlsSecretMapper, r.namespaceResourceMapper, r.clusterSecretResourceMapper, r.applicationSetSCMTLSConfigMapMapper, r.nmMapper) + r.setResourceWatches(bldr, r.clusterResourceMapper, r.tlsSecretMapper, r.namespaceResourceMapper, r.clusterSecretResourceMapper, r.applicationSetSCMTLSConfigMapMapper, r.nmMapper, r.systemCATrustMapper) return bldr.Complete(r) } diff --git a/controllers/argocd/repo_server.go b/controllers/argocd/repo_server.go index d4028b491..15fe0474d 100644 --- a/controllers/argocd/repo_server.go +++ b/controllers/argocd/repo_server.go @@ -19,14 +19,21 @@ import ( "crypto/sha256" "fmt" "reflect" + "strings" "time" appsv1 "k8s.io/api/apps/v1" + "k8s.io/api/certificates/v1beta1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/reconcile" argocdoperatorv1beta1 "github.com/argoproj-labs/argocd-operator/api/v1beta1" "github.com/argoproj-labs/argocd-operator/common" @@ -348,9 +355,10 @@ func (r *ReconcileArgoCD) reconcileRepoDeployment(cr *argocdoperatorv1beta1.Argo if cr.Spec.Repo.Volumes != nil { repoServerVolumes = append(repoServerVolumes, cr.Spec.Repo.Volumes...) } - deploy.Spec.Template.Spec.Volumes = repoServerVolumes + r.injectCATrustToContainers(cr, deploy) + if replicas := getArgoCDRepoServerReplicas(cr); replicas != nil { deploy.Spec.Replicas = replicas } @@ -367,6 +375,12 @@ func (r *ReconcileArgoCD) reconcileRepoDeployment(cr *argocdoperatorv1beta1.Argo } } + log.Info("Applying ArgoCD Repo Server reconciler hook") + if err := applyReconcilerHook(cr, deploy, ""); err != nil { + log.Error(err, "ArgoCD Repo Server reconciler hook failed") + return err + } + existing := newDeploymentWithSuffix("repo-server", "repo-server", cr) deplExists, err := argoutil.IsObjectFound(r.Client, cr.Namespace, existing.Name, existing) if err != nil { @@ -559,6 +573,179 @@ func (r *ReconcileArgoCD) reconcileRepoDeployment(cr *argocdoperatorv1beta1.Argo return r.Create(context.TODO(), deploy) } +// injectCATrustToContainers Creates the init container and volumes to trust CAs specified by `spec.repo.systemCATrust`. +// +// Take CAs from the `argocd-ca-trust-source` volume and mix it with the distro CAs into `argocd-ca-trust-target` volumes. +// Several ubuntu-specific problems exist: +// 1. /etc/ssl/certs/ cannot be updated by `update-ca-certificates` without root - desirable in the production container. +// 2. /etc/ssl/certs/ symlinkes to /usr/local/share/ca-certificates/, so mounting one without the other is futile. +// +// All source certs are projected into the `argocd-ca-trust-source` volume that is ultimately mounted in the prod container (addresses #2). +// +// To amend content of /etc/ssl/certs/ (ca-trust-target), an init container is used: +// - it mounts `argocd-ca-trust-target` over `/etc/ssl/certs/` (addressing #1 by making it writable volume), +// and `ca-trust-source` over `/usr/local/share/ca-certificates/`, +// and amends it with user-added certs using `update-ca-certificates`. +// +// The production container is then mounted with `/etc/ssl/certs/` (`argocd-ca-trust-target`) and +// `/usr/local/share/ca-certificates/` (`argocd-ca-trust-source`) providing read-only CAs needed. +func (r *ReconcileArgoCD) injectCATrustToContainers(cr *argocdoperatorv1beta1.ArgoCD, deploy *appsv1.Deployment) { + if cr.Spec.Repo.SystemCATrust == nil { + return + } + + sources, sourceNames := r.caTrustVolumes(cr) + + volumeSource := "argocd-ca-trust-source" + volumeTarget := "argocd-ca-trust-target" + + repoServerVolumes := []corev1.Volume{ + { + Name: volumeSource, + VolumeSource: corev1.VolumeSource{ + Projected: &corev1.ProjectedVolumeSource{ + Sources: sources, + DefaultMode: ptr.To(int32(0o444)), + }, + }, + }, { + Name: volumeTarget, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + argoImage := getArgoContainerImage(cr) + + deploy.Spec.Template.Spec.InitContainers = append( + deploy.Spec.Template.Spec.InitContainers, + caTrustInitContainer(cr, argoImage, volumeSource, volumeTarget), + ) + + prodVolumeMounts := func() []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: volumeSource, ReadOnly: true, MountPath: "/usr/local/share/ca-certificates/"}, + {Name: volumeTarget, ReadOnly: true, MountPath: "/etc/ssl/certs/"}, + } + } + + // Inject to prod container and sidecars (plugins) + var containerNames []string + for i, container := range deploy.Spec.Template.Spec.Containers { + // This can only work with ubuntu or compatible, so do not inject to potentially incompatible containers + if container.Image == argoImage { + // Accessing by index because the container is a copy of the original struct + deploy.Spec.Template.Spec.Containers[i].VolumeMounts = append(deploy.Spec.Template.Spec.Containers[i].VolumeMounts, prodVolumeMounts()...) + containerNames = append(containerNames, container.Name) + } + } + + log.Info(fmt.Sprintf( + "injecting system CA trust from %s to containers %s", + strings.Join(sourceNames, ", "), + strings.Join(containerNames, ", "), + )) + + deploy.Spec.Template.Spec.Volumes = append(deploy.Spec.Template.Spec.Volumes, repoServerVolumes...) +} + +func (r *ReconcileArgoCD) caTrustVolumes(cr *argocdoperatorv1beta1.ArgoCD) (sources []corev1.VolumeProjection, sourceNames []string) { + // The projected file needs to have the `.crt` suffix for the update-ca-certificates to work correctly. Add it if not present. + ensureValidPath := func(path string) string { + if strings.HasSuffix(path, ".crt") { + return path + } + return path + ".crt" + } + + trackSource := func(kind string, name string, optional *bool) { + path := kind + ":" + name + if optional != nil && *optional { + path += "(optional)" + } + sourceNames = append(sourceNames, path) + } + + for _, bundle := range cr.Spec.Repo.SystemCATrust.ClusterTrustBundles { + bundle = *bundle.DeepCopy() + // Using .Path, because .Name might not be specified + trackSource("ClusterTrustBundle", bundle.Path, bundle.Optional) + + bundle.Path = ensureValidPath(bundle.Path) + sources = append(sources, corev1.VolumeProjection{ClusterTrustBundle: &bundle}) + } + for _, secret := range cr.Spec.Repo.SystemCATrust.Secrets { + secret = *secret.DeepCopy() + trackSource("Secret", secret.Name, secret.Optional) + + for i, item := range secret.Items { + secret.Items[i].Path = ensureValidPath(item.Path) + } + sources = append(sources, corev1.VolumeProjection{Secret: &secret}) + } + for _, cm := range cr.Spec.Repo.SystemCATrust.ConfigMaps { + cm = *cm.DeepCopy() + trackSource("ConfigMap", cm.Name, cm.Optional) + + for i, cmi := range cm.Items { + cm.Items[i].Path = ensureValidPath(cmi.Path) + } + sources = append(sources, corev1.VolumeProjection{ConfigMap: &cm}) + } + return sources, sourceNames +} + +func caTrustInitContainer(cr *argocdoperatorv1beta1.ArgoCD, argoImage string, volumeSource string, volumeTarget string) corev1.Container { + // This is where the image keeps its vendored CAs, look elsewhere if DropImageCertificates + imageCertPath := "/usr/share/ca-certificates" + if cr.Spec.Repo.SystemCATrust.DropImageCertificates { + imageCertPath = "/systemCATrust.dropImageCertificates" + } + + return corev1.Container{ + Name: "update-ca-certificates", + Image: argoImage, + ImagePullPolicy: argoutil.GetImagePullPolicy(cr.Spec.ImagePullPolicy), + Env: proxyEnvVars(corev1.EnvVar{ + Name: "IMAGE_CERT_PATH", + Value: imageCertPath, + }), + Command: []string{"/bin/bash", "-c"}, + Args: []string{` + set -eEuo pipefail + trap 's=$?; echo >&2 "$0: Error on line "$LINENO": $BASH_COMMAND"; exit $s' ERR + + echo "User defined CA files:" + ls -l /usr/local/share/ca-certificates/ + + # Make sure the file exist even when the update-ca-certificates produces no pem blocks + echo "" > /etc/ssl/certs/ca-certificates.crt + + update-ca-certificates --verbose --certsdir "$IMAGE_CERT_PATH" + echo "Resulting /etc/ssl/certs/" + ls -l /etc/ssl/certs/ + echo "Done!" + `}, + VolumeMounts: []corev1.VolumeMount{ + { + Name: volumeSource, + // Source path for user additional certificates - empty in the image, so not shadowing anything. + MountPath: "/usr/local/share/ca-certificates/", + ReadOnly: true, + }, { + Name: volumeTarget, + MountPath: "/etc/ssl/certs/", + }, { + Name: "tmp", + MountPath: "/tmp", + }, + }, + Resources: getArgoRepoResources(cr), + SecurityContext: argoutil.DefaultSecurityContext(), + } +} + // getArgoRepoResources will return the ResourceRequirements for the Argo CD Repo server container. func getArgoRepoResources(cr *argocdoperatorv1beta1.ArgoCD) corev1.ResourceRequirements { resources := corev1.ResourceRequirements{} @@ -759,3 +946,103 @@ func (r *ReconcileArgoCD) reconcileRepoServerTLSSecret(cr *argocdoperatorv1beta1 return nil } + +// systemCATrustMapper triggers reconciliation of repo-server Deployment if some of the tracked Secrets, ConfigMaps or ClusterTrustBundles have changed +func (r *ReconcileArgoCD) systemCATrustMapper(ctx context.Context, o client.Object) []reconcile.Request { + // Track Argo CDs whose repo-servers need a rollout, and id of the resource that changed + rolloutBecause := make(map[*argocdoperatorv1beta1.ArgoCD]string) + + // For cluster-wide resources, it is needed to consult all argos. For cluster-scoped ones, only the argos in the same NS. + argoNamespace := client.InNamespace(o.GetNamespace()) + var argoCDs argocdoperatorv1beta1.ArgoCDList + if err := r.List(ctx, &argoCDs, argoNamespace); err != nil { + log.Error(err, "unable to list ArgoCD instances") + return []reconcile.Request{} + } + + for _, argocd := range argoCDs.Items { + if argocd.Spec.Repo.SystemCATrust == nil { + continue + } + + switch obj := o.(type) { + case *corev1.Secret: + for _, trustSource := range argocd.Spec.Repo.SystemCATrust.Secrets { + if trustSource.Name == obj.Name { + rolloutBecause[&argocd] = fmt.Sprintf("Secret %s/%s", obj.Namespace, obj.Name) + break + } + } + case *corev1.ConfigMap: + for _, trustSource := range argocd.Spec.Repo.SystemCATrust.ConfigMaps { + if trustSource.Name == obj.Name { + rolloutBecause[&argocd] = fmt.Sprintf("ConfigMap %s/%s", obj.Namespace, obj.Name) + break + } + } + case *v1beta1.ClusterTrustBundle: + for _, trustSource := range argocd.Spec.Repo.SystemCATrust.ClusterTrustBundles { + if isRelevantCtb(trustSource, obj) { + rolloutBecause[&argocd] = fmt.Sprintf("ClusterTrustBundle %s", obj.Name) + break + } + } + default: + panic(fmt.Errorf("systemCATrustMapper called for unknown type %t", o)) + } + } + + for argocd, cause := range rolloutBecause { + // Instead of triggering rollout, delete the pod to force trust recomputation + pods := &corev1.PodList{} + err := r.List(context.TODO(), pods, + client.InNamespace(argocd.Namespace), + client.MatchingLabelsSelector{Selector: labels.SelectorFromSet(map[string]string{ + "app.kubernetes.io/name": nameWithSuffix("repo-server", argocd), + })}, + ) + if err != nil { + log.Error(err, "unable to list repo-server pods for argocd", "ns", argocd.Namespace, "name", argocd.Name) + } + + // In normal circumstances, there would be 1 pod. There can be multiple during ongoing rollout. None if not yet started, or recovering from an error. + for _, pod := range pods.Items { + log.Info( + "restarting repo-server pod after SystemCATrust change in "+cause, + "pod", pod.Name, "ns", pod.Namespace, "phase", pod.Status.Phase, + ) + if err := r.Delete(context.TODO(), &pod); err != nil { + log.Error(err, "unable to delete repo-server pod 1", "pod", pod.Name, "ns", pod.Namespace, "phase", pod.Status.Phase) + } + } + } + // No need to reconcile. The pods have been restarted + return []reconcile.Request{} +} + +func isRelevantCtb(proj corev1.ClusterTrustBundleProjection, actual *v1beta1.ClusterTrustBundle) bool { + // ClusterTrustBundle uses either .Name or .SignerName plus eventual .LabelSelector to identify the source + if proj.Name != nil && *proj.Name == actual.Name { + return true + } + + if proj.SignerName != nil && *proj.SignerName == actual.Spec.SignerName { + // If unset, interpreted as "match nothing". If set but empty, interpreted as "match everything". + if proj.LabelSelector == nil { + return false + } + if len(proj.LabelSelector.MatchLabels)+len(proj.LabelSelector.MatchExpressions) == 0 { + return true + } + + selector, err := metav1.LabelSelectorAsSelector(proj.LabelSelector) + if err != nil { + log.Error(err, "Failed evaluating label selector for System CA trust ClusterTrustBundle", "selector", proj.LabelSelector) + return false + } + + return selector.Matches(labels.Set(actual.Labels)) + } + + return false +} diff --git a/controllers/argocd/util.go b/controllers/argocd/util.go index d6c1838c1..6b49bd70e 100644 --- a/controllers/argocd/util.go +++ b/controllers/argocd/util.go @@ -36,6 +36,7 @@ import ( "github.com/argoproj/argo-cd/v3/util/glob" "github.com/distribution/reference" "github.com/go-logr/logr" + certificates "k8s.io/api/certificates/v1beta1" "github.com/argoproj-labs/argocd-operator/api/v1alpha1" argoproj "github.com/argoproj-labs/argocd-operator/api/v1beta1" @@ -75,8 +76,9 @@ const ( ) var ( - versionAPIFound = false - imageUpdaterAPIFound = false + versionAPIFound = false + imageUpdaterAPIFound = false + clusterTrustBundleAPIFound = false ) // IsVersionAPIAvailable returns true if the version api is present @@ -89,6 +91,10 @@ func IsImageUpdaterAPIAvailable() bool { return imageUpdaterAPIFound } +func IsClusterTrustBundleAPIFound() bool { + return clusterTrustBundleAPIFound +} + // verifyVersionAPI will verify that the template API is present. func verifyVersionAPI() error { found, err := argoutil.VerifyAPI(configv1.GroupName, configv1.GroupVersion.Version) @@ -109,6 +115,16 @@ func verifyImageUpdaterAPI() error { return nil } +// verifyClusterTrustBundleAPI will verify that the template API is present. +func verifyClusterTrustBundleAPI() error { + found, err := argoutil.VerifyAPI(certificates.GroupName, certificates.SchemeGroupVersion.Version) + if err != nil { + return err + } + clusterTrustBundleAPIFound = found + return nil +} + // generateArgoAdminPassword will generate and return the admin password for Argo CD. func generateArgoAdminPassword() ([]byte, error) { pass, err := password.Generate( @@ -687,6 +703,9 @@ func InspectCluster() error { if err := verifyVersionAPI(); err != nil { return err } + if err := verifyClusterTrustBundleAPI(); err != nil { + return err + } return nil } @@ -1003,7 +1022,7 @@ func removeString(slice []string, s string) []string { } // setResourceWatches will register Watches for each of the supported Resources. -func (r *ReconcileArgoCD) setResourceWatches(bldr *builder.Builder, clusterResourceMapper, tlsSecretMapper, namespaceResourceMapper, clusterSecretResourceMapper, applicationSetGitlabSCMTLSConfigMapMapper, nmMapper handler.MapFunc) *builder.Builder { +func (r *ReconcileArgoCD) setResourceWatches(bldr *builder.Builder, clusterResourceMapper, tlsSecretMapper, namespaceResourceMapper, clusterSecretResourceMapper, applicationSetGitlabSCMTLSConfigMapMapper, nmMapper, systemCATrustMapper handler.MapFunc) *builder.Builder { // Add new predicate to delete Notifications Resources. The predicate watches the Argo CD CR for changes to the `.spec.Notifications.Enabled` // field. When a change is detected that results in notifications being disabled, we trigger deletion of notifications resources @@ -1031,56 +1050,36 @@ func (r *ReconcileArgoCD) setResourceWatches(bldr *builder.Builder, clusterResou // Watch for changes to primary resource ArgoCD bldr.For(&argoproj.ArgoCD{}, builder.WithPredicates(deleteNotificationsPred, r.argoCDNamespaceManagementFilterPredicate())) - // Watch for changes to ConfigMap sub-resources owned by ArgoCD instances. + // Watch for changes to sub-resources owned by ArgoCD instances. bldr.Owns(&corev1.ConfigMap{}) - - // Watch for changes to Secret sub-resources owned by ArgoCD instances. bldr.Owns(&corev1.Secret{}) - - // Watch for changes to Service sub-resources owned by ArgoCD instances. bldr.Owns(&corev1.Service{}) - - // Watch for changes to Deployment sub-resources owned by ArgoCD instances. bldr.Owns(&appsv1.Deployment{}) - - // Watch for changes to Ingress sub-resources owned by ArgoCD instances. bldr.Owns(&networkingv1.Ingress{}) - + bldr.Owns(&appsv1.StatefulSet{}) bldr.Owns(&v1.Role{}) - bldr.Owns(&v1.RoleBinding{}) + bldr.Owns(&v1alpha1.NotificationsConfiguration{}) - nmMapperResourceHandler := handler.EnqueueRequestsFromMapFunc(nmMapper) - - bldr.Watches(&argoproj.NamespaceManagement{}, nmMapperResourceHandler, builder.WithPredicates(r.namespaceManagementFilterPredicate())) + bldr.Watches(&argoproj.NamespaceManagement{}, handler.EnqueueRequestsFromMapFunc(nmMapper), builder.WithPredicates(r.namespaceManagementFilterPredicate())) clusterResourceHandler := handler.EnqueueRequestsFromMapFunc(clusterResourceMapper) - - clusterSecretResourceHandler := handler.EnqueueRequestsFromMapFunc(clusterSecretResourceMapper) - - appSetGitlabSCMTLSConfigMapHandler := handler.EnqueueRequestsFromMapFunc(applicationSetGitlabSCMTLSConfigMapMapper) - - tlsSecretHandler := handler.EnqueueRequestsFromMapFunc(tlsSecretMapper) - bldr.Watches(&v1.ClusterRoleBinding{}, clusterResourceHandler) - bldr.Watches(&v1.ClusterRole{}, clusterResourceHandler) bldr.Watches(&corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ Name: common.ArgoCDAppSetGitlabSCMTLSCertsConfigMapName, - }}, appSetGitlabSCMTLSConfigMapHandler) + }}, handler.EnqueueRequestsFromMapFunc(applicationSetGitlabSCMTLSConfigMapMapper)) // Watch for secrets of type TLS that might be created by external processes - bldr.Watches(&corev1.Secret{Type: corev1.SecretTypeTLS}, tlsSecretHandler) + bldr.Watches(&corev1.Secret{Type: corev1.SecretTypeTLS}, handler.EnqueueRequestsFromMapFunc(tlsSecretMapper)) // Watch for cluster secrets added to the argocd instance bldr.Watches(&corev1.Secret{ObjectMeta: metav1.ObjectMeta{ Labels: map[string]string{ common.ArgoCDManagedByClusterArgoCDLabel: "cluster", - }}}, clusterSecretResourceHandler) - - // Watch for changes to Secret sub-resources owned by ArgoCD instances. - bldr.Owns(&appsv1.StatefulSet{}) + }, + }}, handler.EnqueueRequestsFromMapFunc(clusterSecretResourceMapper)) // Inspect cluster to verify availability of extra features // This sets the flags that are used in subsequent checks @@ -1101,11 +1100,14 @@ func (r *ReconcileArgoCD) setResourceWatches(bldr *builder.Builder, clusterResou bldr.Owns(&monitoringv1.ServiceMonitor{}) } - // Watch for changes to NotificationsConfiguration CR - bldr.Owns(&v1alpha1.NotificationsConfiguration{}) + systemCATrustHandler := handler.EnqueueRequestsFromMapFunc(systemCATrustMapper) + bldr.Watches(&corev1.Secret{}, systemCATrustHandler) + bldr.Watches(&corev1.ConfigMap{}, systemCATrustHandler) + if IsClusterTrustBundleAPIFound() { + bldr.Watches(&certificates.ClusterTrustBundle{}, systemCATrustHandler) + } namespaceHandler := handler.EnqueueRequestsFromMapFunc(namespaceResourceMapper) - bldr.Watches(&corev1.Namespace{}, namespaceHandler, builder.WithPredicates(r.namespaceFilterPredicate())) bldrHook := newBuilderHook(r.Client, bldr) diff --git a/deploy/olm-catalog/argocd-operator/0.17.0/argoproj.io_argocds.yaml b/deploy/olm-catalog/argocd-operator/0.17.0/argoproj.io_argocds.yaml index adc676874..f0dd40e74 100644 --- a/deploy/olm-catalog/argocd-operator/0.17.0/argoproj.io_argocds.yaml +++ b/deploy/olm-catalog/argocd-operator/0.17.0/argoproj.io_argocds.yaml @@ -20098,6 +20098,237 @@ spec: - name type: object type: array + systemCATrust: + description: Custom certificates to inject into the repo server + container and its plugins to trust source hosting sites + properties: + clusterTrustBundles: + description: ClusterTrustBundles is a list of projected ClusterTrustBundle + volume definitions from where to take the trust certs. + items: + description: |- + ClusterTrustBundleProjection describes how to select a set of + ClusterTrustBundle objects and project their contents into the pod + filesystem. + properties: + labelSelector: + description: |- + Select all ClusterTrustBundles that match this label selector. Only has + effect if signerName is set. Mutually-exclusive with name. If unset, + interpreted as "match nothing". If set but empty, interpreted as "match + everything". + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the + selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: |- + Select a single ClusterTrustBundle by object name. Mutually-exclusive + with signerName and labelSelector. + type: string + optional: + description: |- + If true, don't block pod startup if the referenced ClusterTrustBundle(s) + aren't available. If using name, then the named ClusterTrustBundle is + allowed not to exist. If using signerName, then the combination of + signerName and labelSelector is allowed to match zero + ClusterTrustBundles. + type: boolean + path: + description: Relative path from the volume root to write + the bundle. + type: string + signerName: + description: |- + Select all ClusterTrustBundles that match this signer name. + Mutually-exclusive with name. The contents of all selected + ClusterTrustBundles will be unified and deduplicated. + type: string + required: + - path + type: object + type: array + configMaps: + description: ConfigMaps is a list of projected ConfigMap volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a ConfigMap into a projected volume. + + The contents of the target ConfigMap's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names, + unless the items element is populated with specific mappings of keys to paths. + Note that this is identical to a configmap volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + ConfigMap will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + dropImageCertificates: + description: DropImageCertificates will remove all certs that + are present in the image, leaving only those explicitly + configured here. + type: boolean + secrets: + description: Secrets is a list of projected Secret volume + definitions from where to take the trust certs. + items: + description: |- + Adapts a secret into a projected volume. + + The contents of the target Secret's Data field will be presented in a + projected volume as files using the keys in the Data field as the file names. + Note that this is identical to a secret volume source without the default + mode. + properties: + items: + description: |- + items if unspecified, each key-value pair in the Data field of the referenced + Secret will be projected into the volume as a file whose name is the + key and content is the value. If specified, the listed keys will be + projected into the specified paths, and unlisted keys will not be + present. If a key is specified which is not present in the Secret, + the volume setup will error unless it is marked optional. Paths must be + relative and may not contain the '..' path or start with '..'. + items: + description: Maps a string key to a path within a + volume. + properties: + key: + description: key is the key to project. + type: string + mode: + description: |- + mode is Optional: mode bits used to set permissions on this file. + Must be an octal value between 0000 and 0777 or a decimal value between 0 and 511. + YAML accepts both octal and decimal values, JSON requires decimal values for mode bits. + If not specified, the volume defaultMode will be used. + This might be in conflict with other options that affect the file + mode, like fsGroup, and the result can be other mode bits set. + format: int32 + type: integer + path: + description: |- + path is the relative path of the file to map the key to. + May not be an absolute path. + May not contain the path element '..'. + May not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + x-kubernetes-list-type: atomic + name: + default: "" + description: |- + Name of the referent. + This field is effectively required, but due to backwards compatibility is + allowed to be empty. Instances of this type with an empty value here are + almost certainly wrong. + More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + type: string + optional: + description: optional field specify whether the Secret + or its key must be defined + type: boolean + type: object + x-kubernetes-map-type: atomic + type: array + type: object verifytls: description: VerifyTLS defines whether repo server API should be accessed using strict TLS validation diff --git a/docs/reference/argocd.md b/docs/reference/argocd.md index 8cf852810..7e6af60f3 100644 --- a/docs/reference/argocd.md +++ b/docs/reference/argocd.md @@ -1050,6 +1050,7 @@ Enabled | true | Flag to enable repo server during ArgoCD installation. Remote | [Empty] | Specifies the remote URL of the repo server container. By default, it points to a local instance managed by the operator. This field is optional. Annotations | [Empty] | Custom annotations to pods deployed by the operator Labels | [Empty] | Custom labels to pods deployed by the operator +[SystemCATrust](#repo-server-tls-trust-configuration) | [Empty] | Custom certificates to inject into the repo server container and its plugins to trust source hosting sites ### Pass Command Arguments To Repo Server @@ -1100,6 +1101,52 @@ spec: - 10M ``` +### Repo server TLS trust configuration + +The operator permits injecting custom TLS certificates into the Repo Server container and Config Management Plugins (sidecar containers): + +```yaml +apiVersion: argoproj.io/v1alpha1 +kind: ArgoCD +metadata: + name: example-argocd + labels: + example: repo +spec: + repo: + systemCATrust: + secrets: + - name: my-local-cert-secret + items: + - key: key-name-in-the-secret-object + # Must end with .crt + path: desired-file-name-of-the-certificate.crt + configMaps: + - name: my-local-cert-cm + # Map all keys in the ConfigMap to files with the same name + # Key names in the ConfigMap must end with .crt + items: {} + clusterTrustBundles: + - name: my-global-ctb + path: my-global-ctb.crt + optional: true +``` + +This is orthogonal to declaring per-host TLS certificate in `argocd-tls-certs-cm`, as several notable differences exist: + +- The certificates are not pinned for individual host, so CA/wildcard certificates can be utilized. +- The certificates are properly configured inside the container, permitting more sophisticated (yet secure) plugin logic. For example: + - Kustomize can invoke Helm reaching to other hosts than the source repo + - Kustomize can pull resources from other repositories/sources over HTTPS + - In general, Config Management Plugin can invoke any TLS-enabled tool present in the image with TLS verification on. + +The certificates from Secrets or ConfigMaps must exist in the same namespace as the ArgoCD instance. +Also, they can selectively pick individual keys, or map all their declared keys by omitting the `items` field. + +Each type of the trust source can be declared as optional---the absense of non-optional source will cause deployment failure. + +Unless the `.repo.systemCATrust.dropImageCertificates` is set to true, the user-declared certificates are merged with those from the image. + ## Resource Customizations Resource behavior can be customized using subkeys (`resourceHealthChecks`, `resourceIgnoreDifferences`, and `resourceActions`). Each of the subkeys maps directly to their own field in the `argocd-cm`. `resourceHealthChecks` will map to `resource.customizations.health`, `resourceIgnoreDifferences` to `resource.customizations.ignoreDifferences`, and `resourceActions` to `resource.customizations.actions`. diff --git a/tests/ginkgo/fixture/application/fixture.go b/tests/ginkgo/fixture/application/fixture.go index a2f552be5..bcf56fcf8 100644 --- a/tests/ginkgo/fixture/application/fixture.go +++ b/tests/ginkgo/fixture/application/fixture.go @@ -1,6 +1,9 @@ package application import ( + "fmt" + "regexp" + "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/utils" //lint:ignore ST1001 "This is a common practice in Gomega tests for readability." . "github.com/onsi/gomega" //nolint:all @@ -85,6 +88,40 @@ func HaveSyncStatusCode(expected appv1alpha1.SyncStatusCode) matcher.GomegaMatch } +func HaveNoConditions() matcher.GomegaMatcher { + return expectedCondition(func(app *appv1alpha1.Application) bool { + count := len(app.Status.Conditions) + if count == 0 { + return true + } + + GinkgoWriter.Printf("HaveNoConditions - have: %+v\n", app.Status.Conditions) + return false + }) +} + +func HaveConditionMatching(conditionType appv1alpha1.ApplicationConditionType, messagePattern string) matcher.GomegaMatcher { + pattern := regexp.MustCompile(messagePattern) + + return expectedCondition(func(app *appv1alpha1.Application) bool { + conditions := app.Status.Conditions + var found []string + for _, condition := range conditions { + found = append(found, fmt.Sprintf(" - `%s/%s", condition.Type, condition.Message)) + + if condition.Type == conditionType && pattern.MatchString(condition.Message) { + return true + } + } + + GinkgoWriter.Printf("HaveConditionMatching - expected: `%s/%s; current(%d):\n", conditionType, messagePattern, len(conditions)) + for _, f := range found { + GinkgoWriter.Println(f) + } + return false + }) +} + // Update will keep trying to update object until it succeeds, or times out. func Update(obj *appv1alpha1.Application, modify func(*appv1alpha1.Application)) { k8sClient, _ := utils.GetE2ETestKubeClient() diff --git a/tests/ginkgo/fixture/argocd/fixture.go b/tests/ginkgo/fixture/argocd/fixture.go index 8bcbf7986..85fc62f51 100644 --- a/tests/ginkgo/fixture/argocd/fixture.go +++ b/tests/ginkgo/fixture/argocd/fixture.go @@ -141,32 +141,33 @@ func HaveApplicationControllerOperationProcessors(operationProcessors int) match func HaveCondition(condition metav1.Condition) matcher.GomegaMatcher { return fetchArgoCD(func(argocd *argov1beta1api.ArgoCD) bool { - if len(argocd.Status.Conditions) != 1 { - GinkgoWriter.Println("HaveCondition: length is zero") + length := len(argocd.Status.Conditions) + if length != 1 { + GinkgoWriter.Printf("HaveCondition: length is %d\n", length) return false } instanceCondition := argocd.Status.Conditions[0] - GinkgoWriter.Println("HaveCondition - Message:", instanceCondition.Message, condition.Message) + GinkgoWriter.Printf("HaveCondition - Message: '%s' / actual: '%s'\n", condition.Message, instanceCondition.Message) if instanceCondition.Message != condition.Message { GinkgoWriter.Println("HaveCondition: message does not match") return false } - GinkgoWriter.Println("HaveCondition - Reason:", instanceCondition.Reason, condition.Reason) + GinkgoWriter.Printf("HaveCondition - Reason: '%s' / actual: '%s'\n", condition.Reason, instanceCondition.Reason) if instanceCondition.Reason != condition.Reason { GinkgoWriter.Println("HaveCondition: reason does not match") return false } - GinkgoWriter.Println("HaveCondition - Status:", instanceCondition.Status, condition.Status) + GinkgoWriter.Printf("HaveCondition - Status: '%s' / actual: '%s'\n", condition.Status, instanceCondition.Status) if instanceCondition.Status != condition.Status { GinkgoWriter.Println("HaveCondition: status does not match") return false } - GinkgoWriter.Println("HaveCondition - Type:", instanceCondition.Type, condition.Type) + GinkgoWriter.Printf("HaveCondition - Type: '%s' / actual: '%s'\n", condition.Type, instanceCondition.Type) if instanceCondition.Type != condition.Type { GinkgoWriter.Println("HaveCondition: type does not match") return false diff --git a/tests/ginkgo/fixture/pod/fixture.go b/tests/ginkgo/fixture/pod/fixture.go index 4225dfb11..f3bcee9ab 100644 --- a/tests/ginkgo/fixture/pod/fixture.go +++ b/tests/ginkgo/fixture/pod/fixture.go @@ -2,6 +2,7 @@ package pod import ( "context" + "regexp" "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/utils" //lint:ignore ST1001 "This is a common practice in Gomega tests for readability." @@ -13,6 +14,22 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) +func GetPodByNameRegexp(k8sClient client.Client, nameRegexp *regexp.Regexp, options ...client.ListOption) *corev1.Pod { + var pods []corev1.Pod + list := &corev1.PodList{} + err := k8sClient.List(context.Background(), list, options...) + Expect(err).ToNot(HaveOccurred()) + for _, pod := range list.Items { + if nameRegexp.MatchString(pod.Name) { + pods = append(pods, pod) + } + } + + Expect(pods).Should(HaveLen(1), "expected a single pod matching "+nameRegexp.String()) + + return &pods[0] +} + func GetSpecInitContainerByName(name string, pod corev1.Pod) *corev1.Container { for idx := range pod.Spec.InitContainers { diff --git a/tests/ginkgo/fixture/utils/fixtureUtils.go b/tests/ginkgo/fixture/utils/fixtureUtils.go index 2c6fdda21..3e1f99487 100644 --- a/tests/ginkgo/fixture/utils/fixtureUtils.go +++ b/tests/ginkgo/fixture/utils/fixtureUtils.go @@ -3,6 +3,7 @@ package utils import ( "os" + certificatesv1beta1 "k8s.io/api/certificates/v1beta1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -122,6 +123,10 @@ func getKubeClient(config *rest.Config) (client.Client, *runtime.Scheme, error) return nil, nil, err } + if err := certificatesv1beta1.AddToScheme(scheme); err != nil { + return nil, nil, err + } + k8sClient, err := client.New(config, client.Options{Scheme: scheme}) if err != nil { return nil, nil, err diff --git a/tests/ginkgo/sequential/1-120_repo_server_system_ca_trust.go b/tests/ginkgo/sequential/1-120_repo_server_system_ca_trust.go new file mode 100644 index 000000000..067360c5c --- /dev/null +++ b/tests/ginkgo/sequential/1-120_repo_server_system_ca_trust.go @@ -0,0 +1,852 @@ +/* +Copyright 2025. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sequential + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "fmt" + "regexp" + "strings" + "time" + + "github.com/onsi/gomega/gcustom" + matcher "github.com/onsi/gomega/types" + "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/client-go/util/retry" + "sigs.k8s.io/controller-runtime/pkg/client/apiutil" + + configmapFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/configmap" + secretFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/secret" + + "k8s.io/utils/ptr" + + appFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/application" + osFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/os" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + certificatesv1alpha1 "k8s.io/api/certificates/v1beta1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + appv1alpha1 "github.com/argoproj/argo-cd/v3/pkg/apis/application/v1alpha1" + + argov1beta1api "github.com/argoproj-labs/argocd-operator/api/v1beta1" + "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture" + argocdFixture "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/argocd" + fixtureUtils "github.com/argoproj-labs/argocd-operator/tests/ginkgo/fixture/utils" +) + +var ( + // The differences between the upstream image using Ubuntu, and the downstream one using rhel. + image = "" // argocd-operator default + version = "" // argocd-operator default + caBundlePath = "/etc/ssl/certs/ca-certificates.crt" + + trustedHelmAppSource = &appv1alpha1.ApplicationSource{ + RepoURL: "https://stefanprodan.github.io/podinfo", + Chart: "podinfo", + TargetRevision: "6.5.3", + Helm: &appv1alpha1.ApplicationSourceHelm{Values: ""}, + } + + untrustedHelmAppSource = &appv1alpha1.ApplicationSource{ + RepoURL: "https://helm.nginx.com/stable", + Chart: "nginx", + TargetRevision: "1.1.0", + Helm: &appv1alpha1.ApplicationSourceHelm{Values: "service:\n type: ClusterIP"}, + } + + k8sClient client.Client + ctx context.Context + + clusterSupportsClusterTrustBundles bool +) + +var _ = Describe("GitOps Operator Sequential E2E Tests", func() { + + Context("1-120_repo_server_system_ca_trust", func() { + BeforeEach(func() { + fixture.EnsureSequentialCleanSlate() + + k8sClient, _ = fixtureUtils.GetE2ETestKubeClient() + ctx = context.Background() + + clusterSupportsClusterTrustBundles = detectClusterTrustBundleSupport(k8sClient, ctx) + }) + + AfterEach(func() { + purgeCtbs() + }) + + It("ensures that missing Secret aborts startup", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + By("creating Argo CD instance with missing Secret") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + Secrets: []corev1.SecretProjection{ + {LocalObjectReference: corev1.LocalObjectReference{Name: "no-such-secret"}}, + }, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + + Eventually(argoCD, "1m", "5s").Should(argocdFixture.HaveServerStatus("Running")) + Consistently(argoCD, "20s", "5s").Should(argocdFixture.HaveRepoStatus("Pending")) + Expect(argoCD).ShouldNot(argocdFixture.BeAvailable()) + }) + + It("ensures that ClusterTrustBundles are trusted in repo-server and plugins", func() { + if !clusterSupportsClusterTrustBundles { + Skip("Cluster does not support ClusterTrustBundles") + } + + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + // Create a bundle with 2 CA certs in it. Ubuntu's update-ca-certificates issues a warning, but apparently it works + // It is desirable to test with multiple certs in one bundle because OpenShift permits it + combinedCtb := createCtbFromCerts(getCACert("github.com"), getCACert("github.io")) + _ = k8sClient.Delete(ctx, combinedCtb) // Exists only in case of previous failures + defer func() { _ = k8sClient.Delete(ctx, combinedCtb) }() + Expect(k8sClient.Create(ctx, combinedCtb)).To(Succeed()) + + pluginCm, pluginContainer, pluginVolumes := createGitPullingPlugin(ns) + Expect(k8sClient.Create(ctx, pluginCm)).To(Succeed()) + + By("creating Argo CD instance trusting CTBs") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // So we can test against upstream sites that would otherwise be trusted by the image + ClusterTrustBundles: []corev1.ClusterTrustBundleProjection{ + {Name: ptr.To(combinedCtb.Name), Path: "combined.crt"}, + {Name: ptr.To("nah"), Path: "no-such-ctb.crt", Optional: ptr.To(true)}, + }, + }, + // plugin containers/volumes - this is not related to CTBs + Volumes: pluginVolumes, + SidecarContainers: []corev1.Container{ + *pluginContainer, + }, + }) + + By("verifying correctly established system trust") + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + + verifyCorrectlyConfiguredTrust(ns) + Expect(repoServerSystemCaTrust(ns)).Should(trustCerts(Equal(2), And( + ContainSubstring("combined.crt"), + ContainSubstring("no-such-ctb.crt"), + ))) + }) + + It("ensures that CMs and Secrets are trusted in repo-server and plugins", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + cmCert := createCmFromCert(ns, getCACert("github.com")) + Expect(k8sClient.Create(ctx, cmCert)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, cmCert) }() + secretCert := createSecretFromCert(ns, getCACert("github.io")) + Expect(k8sClient.Create(ctx, secretCert)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, secretCert) }() + + pluginCm, pluginContainer, pluginVolumes := createGitPullingPlugin(ns) + Expect(k8sClient.Create(ctx, pluginCm)).To(Succeed()) + + By("creating Argo CD instance trusting CTBs") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // So we can test against upstream sites that would otherwise be trusted by the image + Secrets: []corev1.SecretProjection{{ + // No Items, Map all + LocalObjectReference: corev1.LocalObjectReference{ + Name: secretCert.Name, + }, + }}, + ConfigMaps: []corev1.ConfigMapProjection{{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: cmCert.Name, + }, + Optional: ptr.To(true), + Items: []corev1.KeyToPath{ + {Key: "ca.cm.crt", Path: "ca.cm.wrong-suffix"}, + }, + }}, + }, + // plugin containers/volumes - this is not related to Secret/CM + Volumes: pluginVolumes, + SidecarContainers: []corev1.Container{ + *pluginContainer, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + + initContainerLog := getRepoCertGenerationLog(findRunningRepoServerPod(k8sClient, ns)) + Expect(initContainerLog).Should(ContainSubstring("ca.secret.crt")) + Expect(initContainerLog).Should(ContainSubstring("ca.cm.wrong-suffix.crt")) + verifyCorrectlyConfiguredTrust(ns) + }) + + It("ensures that 0 trusted certs with DropImageCertificates trusts nothing", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + By("creating Argo CD instance with empty system trust") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + + Expect(repoServerSystemCaTrust(ns)).Should(trustCerts(Equal(0), Not(BeEmpty()))) + + trustedHelmApp := createHelmApp(ns, trustedHelmAppSource) + Expect(k8sClient.Create(ctx, trustedHelmApp)).To(Succeed()) + + // Sleep to make sure the apps sync took place - otherwise there might be no conditions _yet_ + time.Sleep(20 * time.Second) + + Expect(trustedHelmApp).Should(appFixture.HaveConditionMatching( + "ComparisonError", + ".*tls: failed to verify certificate: x509: certificate signed by unknown authority.*", + )) + Expect(trustedHelmApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeUnknown)) + }) + + It("ensures that empty trust keeps image certs in place", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + By("creating Argo CD instance with empty system trust") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: false, // Keep the image ones + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + Expect(repoServerSystemCaTrust(ns)).Should(trustCerts(BeNumerically(">", 100), Not(BeEmpty()))) + }) + + It("ensures that Secrets and ConfigMaps get reconciled", func() { + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + By("creating Argo CD instance with empty system trust, but full of anticipation") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // To make the counting easier + Secrets: []corev1.SecretProjection{{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "ca-trust", + }, + Optional: ptr.To(true), + }}, + ConfigMaps: []corev1.ConfigMapProjection{{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "ca-trust", + }, + Optional: ptr.To(true), + }}, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + actualTrust := repoServerSystemCaTrust(ns) + Expect(actualTrust).Should(trustCerts(Equal(0), Not(BeEmpty()))) + + By("creating ConfigMap with 1 cert") + cmCert := createCmFromCert(ns, getCACert("github.com")) + Expect(k8sClient.Create(ctx, cmCert)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, cmCert) }() + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(1), Not(BeEmpty()))) + + By("creating Secret with 1 cert") + secretCert := createSecretFromCert(ns, getCACert("github.io")) + Expect(k8sClient.Create(ctx, secretCert)).To(Succeed()) + defer func() { _ = k8sClient.Delete(ctx, secretCert) }() + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(2), Not(BeEmpty()))) + + By("updating ConfigMap to 2 certs") + configmapFixture.Update(cmCert, func(configMap *corev1.ConfigMap) { + configMap.Data = map[string]string{ + "a.crt": getCACert("github.com"), + "b.crt": getCACert("google.com"), + } + }) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(3), Not(BeEmpty()))) + + By("updating Secret to 0 certs") + secretFixture.Update(secretCert, func(secret *corev1.Secret) { + // Albeit `.Data` is never written by the test, it is the field that holds the data after the Create/Get roundtrip. + // Erase, otherwise reducing the content of `.StringData` does not have the expected effect. + secret.Data = map[string][]byte{} + secret.StringData = map[string]string{} + }) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(2), Not(BeEmpty()))) + + By("updating ConfigMap to 1 certs") + configmapFixture.Update(cmCert, func(configMap *corev1.ConfigMap) { + configMap.Data = map[string]string{ + "a.crt": getCACert("redhat.com"), + } + }) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(1), Not(BeEmpty()))) + + By("deleting ConfigMap") + Expect(k8sClient.Delete(ctx, cmCert)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(0), Not(BeEmpty()))) + }) + + It("ensures that ClusterTrustBundles get reconciled", func() { + if !clusterSupportsClusterTrustBundles { + Skip("Cluster does not support ClusterTrustBundles") + } + + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + combinedCtb := createCtbFromCerts(getCACert("github.com"), getCACert("github.io")) + _ = k8sClient.Delete(ctx, combinedCtb) // Exists only in case of previous failures, must be deleted before argo starts! + + By("creating Argo CD instance with empty system trust, but full of anticipation") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // To make the counting easier + ClusterTrustBundles: []corev1.ClusterTrustBundleProjection{{ + Name: ptr.To(combinedCtb.Name), Path: "ctb.crt", Optional: ptr.To(true), + }}, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + actualTrust := repoServerSystemCaTrust(ns) + Expect(actualTrust).Should(trustCerts(Equal(0), Not(BeEmpty())), actualTrust.diagnose()) + + By("creating ClusterTrustBundle with 2 certs") + defer func() { _ = k8sClient.Delete(ctx, combinedCtb) }() + Expect(k8sClient.Create(ctx, combinedCtb)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(2), Not(BeEmpty())), actualTrust.diagnose()) + + By("updating ClusterTrustBundle with 1 cert") + ctbUpdate(combinedCtb, func(bundle *certificatesv1alpha1.ClusterTrustBundle) { + bundle.Spec = certificatesv1alpha1.ClusterTrustBundleSpec{ + SignerName: bundle.Spec.SignerName, + TrustBundle: getCACert("github.com"), + } + }) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "6m", "15s").Should(trustCerts(Equal(1), Not(BeEmpty())), actualTrust.diagnose()) + + By("deleting ClusterTrustBundle") + Expect(k8sClient.Delete(ctx, combinedCtb)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "6m", "15s").Should(trustCerts(Equal(0), Not(BeEmpty())), actualTrust.diagnose()) + By("done") + }) + + It("detect only relevant ClusterTrustBundles changes", func() { + if !clusterSupportsClusterTrustBundles { + Skip("Cluster does not support ClusterTrustBundles") + } + + ns, cleanupFunc := fixture.CreateRandomE2ETestNamespaceWithCleanupFunc() + defer cleanupFunc() + + // Use random label value not to collide with leftover CTBs fom other tests + labelVal := rand.String(5) + signerName := "acme.com/signer" + By("creating Argo CD instance with system trust") + argoCD := argoCDSpec(ns, argov1beta1api.ArgoCDRepoSpec{ + SystemCATrust: &argov1beta1api.ArgoCDSystemCATrustSpec{ + DropImageCertificates: true, // To make the counting easier + // Test CTB update detection based on CTB binding specified by labels - no real signers involved + ClusterTrustBundles: []corev1.ClusterTrustBundleProjection{ + { + SignerName: ptr.To(signerName), + LabelSelector: &metav1.LabelSelector{MatchLabels: map[string]string{ + "test": labelVal, + }}, + Path: "one.crt", + Optional: ptr.To(true), + }, + }, + }, + }) + Expect(k8sClient.Create(ctx, argoCD)).To(Succeed()) + Eventually(argoCD, "5m", "5s").Should(argocdFixture.BeAvailable()) + + By("adding ClusterTrustBundle with 1 cert") + oneCtb := createCtbFromCerts(getCACert("github.com")) + oneCtb.Labels["test"] = labelVal + oneCtb.Name = "acme.com:signer:repo-server-system-ca-trust-test-one" + oneCtb.Spec.SignerName = signerName + Expect(k8sClient.Create(ctx, oneCtb)).To(Succeed()) + actualTrust := repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(1), Not(BeEmpty())), actualTrust.diagnose()) + + By("adding ClusterTrustBundle with other cert") + twoCtb := createCtbFromCerts(getCACert("github.io")) + twoCtb.Labels["test"] = labelVal + twoCtb.Name = "acme.com:signer:repo-server-system-ca-trust-test-two" + twoCtb.Spec.SignerName = signerName + Expect(k8sClient.Create(ctx, twoCtb)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Eventually(actualTrust, "30s", "5s").Should(trustCerts(Equal(2), Not(BeEmpty())), actualTrust.diagnose()) + + By("updating Argo CD to read from ClusterTrustBundle that does not exist") + argocdFixture.Update(argoCD, func(cd *argov1beta1api.ArgoCD) { + cd.Spec.Repo.SystemCATrust.ClusterTrustBundles = []corev1.ClusterTrustBundleProjection{ + { + Name: ptr.To("no-such-ctb"), + Path: "three.crt", + Optional: ptr.To(true), + }, + } + }) + actualTrust = repoServerSystemCaTrust(ns) + Consistently(actualTrust, "10s", "5s").Should(trustCerts(Equal(0), Not(BeEmpty())), actualTrust.diagnose()) + oldPodName := findRunningRepoServerPod(k8sClient, ns).Name + + By("creating unrelated ClusterTrustBundle") + fourCtb := createCtbFromCerts(getCACert("google.com")) + Expect(k8sClient.Create(ctx, fourCtb)).To(Succeed()) + actualTrust = repoServerSystemCaTrust(ns) + Consistently(actualTrust, "10s", "5s").Should(trustCerts(Equal(0), Not(BeEmpty())), actualTrust.diagnose()) + newPodName := findRunningRepoServerPod(k8sClient, ns).Name + Expect(newPodName).To(Equal(oldPodName), "Pod have restarted for unrelated change") + }) + }) +}) + +func ctbUpdate(obj *certificatesv1alpha1.ClusterTrustBundle, modify func(*certificatesv1alpha1.ClusterTrustBundle)) { + err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + // Retrieve the latest version of the object + err := k8sClient.Get(context.Background(), client.ObjectKeyFromObject(obj), obj) + if err != nil { + return err + } + + modify(obj) + + // Attempt to update the object + return k8sClient.Update(context.Background(), obj) + }) + Expect(err).ToNot(HaveOccurred()) +} + +func argoCDSpec(ns *corev1.Namespace, repoSpec argov1beta1api.ArgoCDRepoSpec) *argov1beta1api.ArgoCD { + return &argov1beta1api.ArgoCD{ + ObjectMeta: metav1.ObjectMeta{Name: "argocd", Namespace: ns.Name}, + Spec: argov1beta1api.ArgoCDSpec{ + Image: image, + Version: version, + Repo: repoSpec, + }, + } +} + +func detectClusterTrustBundleSupport(k8sClient client.Client, ctx context.Context) bool { + err := k8sClient.List(ctx, &certificatesv1alpha1.ClusterTrustBundleList{}) + if _, ok := err.(*apiutil.ErrResourceDiscoveryFailed); ok { + return false + } + Expect(err).ToNot(HaveOccurred()) // Every other error is an error + return true +} + +func createGitPullingPlugin(ns *corev1.Namespace) (*corev1.ConfigMap, *corev1.Container, []corev1.Volume) { + By("Creating ConfigManagementPlugin resources for git clone") + name := "cmp-git-https" + cm := &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + Labels: map[string]string{ + "app.kubernetes.io/name": name, + "app.kubernetes.io/part-of": "argocd", + }, + }, + Data: map[string]string{ + "plugin.yaml": `apiVersion: argoproj.io/v1alpha1 +kind: ConfigManagementPlugin +metadata: + name: git-https +spec: + version: v1.0 + generate: + command: [bash, -c] + args: + - | + set -euxo pipefail + git clone --depth 1 --verbose "$ARGOCD_APP_SOURCE_REPO_URL" +`, + }, + } + + container := &corev1.Container{ + Name: name, + Command: []string{"/var/run/argocd/argocd-cmp-server"}, + VolumeMounts: []corev1.VolumeMount{ + { + MountPath: "/var/run/argocd", + Name: "var-files", + }, + { + MountPath: "/home/argocd/cmp-server/plugins", + Name: "plugins", + }, + { + MountPath: "/home/argocd/cmp-server/config/plugin.yaml", + SubPath: "plugin.yaml", + Name: name + "-config", + }, + { + MountPath: "/tmp", + Name: name + "-tmp", + }, + }, + } + + volumes := []corev1.Volume{ + { + Name: name + "-config", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: name, + }, + }, + }, + }, + { + Name: name + "-tmp", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + } + + return cm, container, volumes +} + +func createHelmApp(ns *corev1.Namespace, source *appv1alpha1.ApplicationSource) *appv1alpha1.Application { + By("creating helm Application " + source.Chart) + + return &appv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: source.Chart, + Namespace: ns.Name, + }, + Spec: appv1alpha1.ApplicationSpec{ + Project: "default", + Source: source, + Destination: appv1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: ns.Name, + }, + SyncPolicy: &appv1alpha1.SyncPolicy{ + Automated: &appv1alpha1.SyncPolicyAutomated{ + Prune: true, SelfHeal: true, + }, + }, + }, + } +} + +func createPluginApp(ns *corev1.Namespace, url string) *appv1alpha1.Application { + name := regexp.MustCompile("[^a-z]+").ReplaceAllString(url, "-") + By("creating plugin Application " + name) + return &appv1alpha1.Application{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: ns.Name, + }, + Spec: appv1alpha1.ApplicationSpec{ + Project: "default", + Destination: appv1alpha1.ApplicationDestination{ + Server: "https://kubernetes.default.svc", + Namespace: ns.Name, + }, + Source: &appv1alpha1.ApplicationSource{ + RepoURL: url, + TargetRevision: "HEAD", + Path: ".", + Plugin: &appv1alpha1.ApplicationSourcePlugin{ + Name: "git-https-v1.0", + Env: appv1alpha1.Env{ + &appv1alpha1.EnvEntry{ + Name: "ARGOCD_APP_SOURCE_REPO_URL", + Value: url, + }, + }, + }, + }, + }, + } +} + +func createCtbFromCerts(bundle ...string) *certificatesv1alpha1.ClusterTrustBundle { + return &certificatesv1alpha1.ClusterTrustBundle{ + TypeMeta: metav1.TypeMeta{ + Kind: "ClusterTrustBundle", + APIVersion: "certificates.k8s.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "repo-server-system-ca-trust", + Labels: map[string]string{ + "argocd-operator-test": "repo_server_system_ca_trust", + }, + }, + Spec: certificatesv1alpha1.ClusterTrustBundleSpec{ + TrustBundle: strings.Join(bundle, "\n"), + }, + } +} + +func createCmFromCert(ns *corev1.Namespace, bundle string) *corev1.ConfigMap { + return &corev1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ca-trust", + Namespace: ns.Name, + }, + Data: map[string]string{ + "ca.cm.crt": bundle, + }, + } +} + +func createSecretFromCert(ns *corev1.Namespace, bundle string) *corev1.Secret { + return &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ca-trust", + Namespace: ns.Name, + }, + Type: "Opaque", + StringData: map[string]string{ + "ca.secret.crt": bundle, + }, + } +} + +func getCACert(host string) string { + config := &tls.Config{MinVersion: tls.VersionTLS13} + conn, err := tls.Dial("tcp", host+":443", config) + Expect(err).ToNot(HaveOccurred()) + defer func() { _ = conn.Close() }() + + pcs := conn.ConnectionState().PeerCertificates + + // ClusterTrustBundle cannot hold leaf certificates, so testing with CA cert at least. In theory, some of the hosts + // we test against can share the same CA cert, so albeit not likely, rudimentary negative testing is needed. + return encodeCert(pcs[len(pcs)-1]) +} + +func encodeCert(cert *x509.Certificate) string { + writer := strings.Builder{} + err := pem.Encode(&writer, &pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}) + Expect(err).ToNot(HaveOccurred()) + + return writer.String() +} + +type podTrust struct { + ns *corev1.Namespace + k8sClient client.Client + + count int + log string + events string +} + +func (pt *podTrust) fetch() { + pod := findRunningRepoServerPod(pt.k8sClient, pt.ns) + pt.count = getTrustedCertCount(pod) + pt.log = getRepoCertGenerationLog(pod) + + out, err := osFixture.ExecCommandWithOutputParam(false, "kubectl", "-n", pt.ns.Name, "events") + if err != nil { + panic(err) + } + pt.events = out +} + +func (pt *podTrust) diagnose() string { + return fmt.Sprintf( + "System CA Trust init contianer log:\n%s\nProject events:\n%s\n", + pt.log, pt.events, + ) +} + +func repoServerSystemCaTrust(ns *corev1.Namespace) *podTrust { + return &podTrust{ns: ns, k8sClient: k8sClient} +} + +func trustCerts(countMatcher, logMatcher matcher.GomegaMatcher) matcher.GomegaMatcher { + // Wrap to capture and attach diagnostics + matchCount := gcustom.MakeMatcher(func(pt *podTrust) (bool, error) { + // call fetch exactly once before matchCount _and_ matchLog + pt.fetch() + + success, err := countMatcher.Match(pt.count) + if err != nil { + return false, err + } + if success { + return true, nil + } + + return false, fmt.Errorf( + "%s\n\n--- Diagnostics ---\n%s\n===", + countMatcher.FailureMessage(pt.count), + pt.diagnose(), + ) + }) + + matchLog := WithTransform(func(pt *podTrust) string { + return pt.log + }, logMatcher) + + return And(matchCount, matchLog) +} + +func getTrustedCertCount(rsPod *corev1.Pod) int { + var out string + var err error + // retry a few times, because pod can be restarting during trust source update, get Terminating between check and use. + for i := 0; i < 3; i++ { + command := []string{ + "kubectl", "-n", rsPod.Namespace, "exec", + "-c", "argocd-repo-server", rsPod.Name, "--", + "cat", caBundlePath, + } + + out, err = osFixture.ExecCommandWithOutputParam(false, command...) + if err == nil { + break + } + time.Sleep(1 * time.Second) + } + Expect(err).ToNot(HaveOccurred(), out) + + seen := make(map[string]bool) + var currentBlock strings.Builder + for line := range strings.Lines(out) { + switch { + case strings.Contains(line, "BEGIN CERTIFICATE"): + currentBlock.Reset() + case strings.Contains(line, "END CERTIFICATE"): + seen[currentBlock.String()] = true + default: + currentBlock.WriteString(line) + } + } + return len(seen) +} + +func getRepoCertGenerationLog(rsPod *corev1.Pod) string { + out, err := osFixture.ExecCommandWithOutputParam( + false, + "kubectl", "-n", rsPod.Namespace, "logs", "-c", "update-ca-certificates", rsPod.Name, + ) + Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("output: %s", out)) + return out +} + +func findRunningRepoServerPod(k8sClient client.Client, ns *corev1.Namespace) *corev1.Pod { + nameRegexp := regexp.MustCompile(".*-repo-server.*") + var pods []*corev1.Pod + + for j := 0; j < 10; j++ { + time.Sleep(2 * time.Second) + pods = []*corev1.Pod{} + list := &corev1.PodList{} + err := k8sClient.List(context.Background(), list, client.InNamespace(ns.Name)) + Expect(err).ToNot(HaveOccurred()) + + for _, pod := range list.Items { + if pod.Status.Phase == "Running" && nameRegexp.MatchString(pod.Name) { + pods = append(pods, &pod) + } + } + if len(pods) == 1 { + return pods[0] + } + } + + panic(fmt.Sprintf("Failed to find Running repo-server pod. have %+v", pods)) +} + +func verifyCorrectlyConfiguredTrust(ns *corev1.Namespace) { + untrustedHelmApp := createHelmApp(ns, untrustedHelmAppSource) + Expect(k8sClient.Create(ctx, untrustedHelmApp)).To(Succeed()) + + // Using some host not trusted by github's intermediate cert. Gitlab-somewhat surprisingly-is. + untrustedPluginApp := createPluginApp(ns, "https://kernel.googlesource.com/pub/scm/docs/man-pages/website.git") + Expect(k8sClient.Create(ctx, untrustedPluginApp)).To(Succeed()) + + trustedHelmApp := createHelmApp(ns, trustedHelmAppSource) + Expect(k8sClient.Create(ctx, trustedHelmApp)).To(Succeed()) + + trustedPluginApp := createPluginApp(ns, "https://github.com/argoproj-labs/argocd-operator.git") + Expect(k8sClient.Create(ctx, trustedPluginApp)).To(Succeed()) + + // Sleep to make sure the apps sync took place - otherwise there might be no conditions _yet_ + time.Sleep(20 * time.Second) + + Expect(untrustedHelmApp).Should( + appFixture.HaveConditionMatching("ComparisonError", ".*failed to fetch chart.*"), + ) + Expect(untrustedHelmApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeUnknown)) + + Expect(untrustedPluginApp).Should( + appFixture.HaveConditionMatching("ComparisonError", ".*certificate signed by unknown authority.*"), + ) + Expect(untrustedPluginApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeUnknown)) + + Expect(trustedHelmApp).Should(appFixture.HaveNoConditions()) + Expect(trustedHelmApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeSynced)) + + Expect(trustedPluginApp).Should(appFixture.HaveNoConditions()) + Expect(trustedPluginApp).Should(appFixture.HaveSyncStatusCode(appv1alpha1.SyncStatusCodeSynced)) +} + +// purgeCtbs deletes all of the cluster-wide resource, that can get leaked on test failure/abort. +func purgeCtbs() { + if clusterSupportsClusterTrustBundles { + expr := client.MatchingLabels{"argocd-operator-test": "repo_server_system_ca_trust"} + Expect(k8sClient.DeleteAllOf(ctx, &certificatesv1alpha1.ClusterTrustBundle{}, expr)).To(Succeed()) + } +}