diff --git a/e2e-tests/arbiter/run b/e2e-tests/arbiter/run index 2721feae74..fe2bbb22f5 100755 --- a/e2e-tests/arbiter/run +++ b/e2e-tests/arbiter/run @@ -71,7 +71,7 @@ check_cr_config() { main() { create_infra $namespace - deploy_cert_manager + deploy_cert_manager "--enable-certificate-owner-ref" desc 'create secrets and start client' kubectl_bin apply \ diff --git a/e2e-tests/functions b/e2e-tests/functions index 9db299bd31..51b4507e25 100755 --- a/e2e-tests/functions +++ b/e2e-tests/functions @@ -885,7 +885,6 @@ run_mongosh() { bash -c "printf '$command\n' | mongosh --quiet $driver://$uri$suffix/admin?ssl=false\&replicaSet=$replica_set $mongo_flag" } - run_mongo_tls() { local command="$1" local uri="$2" @@ -1158,6 +1157,10 @@ deploy_cert_manager() { kubectl_bin create namespace cert-manager || : kubectl_bin label namespace cert-manager certmanager.k8s.io/disable-validation=true || : kubectl_bin apply -f "https://github.com/cert-manager/cert-manager/releases/download/v${CERT_MANAGER_VER}/cert-manager.yaml" --validate=false || : 2>/dev/null + for arg in "$@"; do + kubectl_bin patch deployment cert-manager -n cert-manager --type='json' \ + -p='[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"'"$arg"'"}]' + done kubectl_bin -n cert-manager wait pod -l app.kubernetes.io/instance=cert-manager --for=condition=ready sleep 120 } diff --git a/pkg/controller/perconaservermongodb/ssl.go b/pkg/controller/perconaservermongodb/ssl.go index 105193e00a..c618f8fd9c 100644 --- a/pkg/controller/perconaservermongodb/ssl.go +++ b/pkg/controller/perconaservermongodb/ssl.go @@ -191,7 +191,7 @@ func (r *ReconcilePerconaServerMongoDB) createSSLByCertManager(ctx context.Conte return nil } - caSecret, err := r.getSecret(ctx, cr, tls.CACertificateSecretName(cr)) + caSecret, err := r.getSecret(ctx, cr, tls.CertificateCA(cr).SecretName()) if err != nil { if k8serr.IsNotFound(err) { return nil @@ -393,14 +393,15 @@ func (r *ReconcilePerconaServerMongoDB) applyCertManagerCertificates(ctx context return "", errors.Wrap(err, "apply ca issuer") } + caCert := tls.CertificateCA(cr) err = applyFunc(func() (util.ApplyStatus, error) { - return c.ApplyCACertificate(ctx, cr) + return c.ApplyCertificate(ctx, cr, caCert) }) if err != nil { return "", errors.Wrap(err, "create ca certificate") } - err = c.WaitForCerts(ctx, cr, tls.CACertificateSecretName(cr)) + err = c.WaitForCerts(ctx, cr, caCert) if err != nil { return "", errors.Wrap(err, "failed to wait for ca cert") } @@ -413,26 +414,27 @@ func (r *ReconcilePerconaServerMongoDB) applyCertManagerCertificates(ctx context return "", errors.Wrap(err, "create issuer") } + tlsCert := tls.CertificateTLS(cr, false) err = applyFunc(func() (util.ApplyStatus, error) { - return c.ApplyCertificate(ctx, cr, false) + return c.ApplyCertificate(ctx, cr, tlsCert) }) if err != nil { return "", errors.Wrap(err, "create certificate") } - secretNames := []string{tls.CertificateSecretName(cr, false)} + certificates := []tls.Certificate{tlsCert} - if tls.CertificateSecretName(cr, false) != tls.CertificateSecretName(cr, true) { + if internalCert := tls.CertificateTLS(cr, true); tlsCert.SecretName() != internalCert.SecretName() { err = applyFunc(func() (util.ApplyStatus, error) { - return c.ApplyCertificate(ctx, cr, true) + return c.ApplyCertificate(ctx, cr, internalCert) }) if err != nil { return "", errors.Wrap(err, "create certificate") } - secretNames = append(secretNames, tls.CertificateSecretName(cr, true)) + certificates = append(certificates, internalCert) } - err = c.WaitForCerts(ctx, cr, secretNames...) + err = c.WaitForCerts(ctx, cr, certificates...) if err != nil { return "", errors.Wrap(err, "failed to wait for certs") } diff --git a/pkg/psmdb/tls/certificate.go b/pkg/psmdb/tls/certificate.go new file mode 100644 index 0000000000..d0d36bf688 --- /dev/null +++ b/pkg/psmdb/tls/certificate.go @@ -0,0 +1,135 @@ +package tls + +import ( + "time" + + cm "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" + cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + api "github.com/percona/percona-server-mongodb-operator/pkg/apis/psmdb/v1" + "github.com/percona/percona-server-mongodb-operator/pkg/naming" +) + +type Certificate interface { + Name() string + SecretName() string + Object() *cm.Certificate +} + +type caCert struct { + cr *api.PerconaServerMongoDB +} + +func CertificateCA(cr *api.PerconaServerMongoDB) Certificate { + return &caCert{ + cr: cr, + } +} + +func (c *caCert) Name() string { + return c.cr.Name + "-ca-cert" +} + +func (c *caCert) SecretName() string { + return c.Name() +} + +func (c *caCert) Object() *cm.Certificate { + cr := c.cr + + labels := naming.ClusterLabels(cr) + if cr.CompareVersion("1.17.0") < 0 { + labels = nil + } + return &cm.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.Name(), + Namespace: cr.Namespace, + Labels: labels, + }, + Spec: cm.CertificateSpec{ + SecretName: c.SecretName(), + CommonName: cr.Name + "-ca", + IsCA: true, + IssuerRef: cmmeta.ObjectReference{ + Name: caIssuerName(cr), + Kind: cm.IssuerKind, + }, + Duration: &metav1.Duration{Duration: time.Hour * 24 * 365}, + RenewBefore: &metav1.Duration{Duration: 730 * time.Hour}, + }, + } +} + +type tlsCert struct { + cr *api.PerconaServerMongoDB + + internal bool +} + +func CertificateTLS(cr *api.PerconaServerMongoDB, internal bool) Certificate { + return &tlsCert{ + cr: cr, + internal: internal, + } +} + +func (c *tlsCert) Name() string { + if c.internal { + return c.cr.Name + "-ssl-internal" + } + return c.cr.Name + "-ssl" +} + +func (c *tlsCert) SecretName() string { + if c.internal { + return api.SSLInternalSecretName(c.cr) + } + + return api.SSLSecretName(c.cr) +} + +func (c *tlsCert) Object() *cm.Certificate { + cr := c.cr + + issuerKind := cm.IssuerKind + issuerGroup := "" + if cr.CompareVersion("1.16.0") >= 0 && cr.Spec.TLS != nil && cr.Spec.TLS.IssuerConf != nil { + issuerKind = cr.Spec.TLS.IssuerConf.Kind + issuerGroup = cr.Spec.TLS.IssuerConf.Group + + } + isCA := false + if cr.CompareVersion("1.15.0") < 0 { + isCA = true + } + + labels := naming.ClusterLabels(cr) + if cr.CompareVersion("1.17.0") < 0 { + labels = nil + } + + return &cm.Certificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: c.Name(), + Namespace: cr.Namespace, + Labels: labels, + }, + Spec: cm.CertificateSpec{ + Subject: &cm.X509Subject{ + Organizations: []string{"PSMDB"}, + }, + CommonName: cr.Name, + SecretName: c.SecretName(), + DNSNames: GetCertificateSans(cr), + IsCA: isCA, + Duration: &cr.Spec.TLS.CertValidityDuration, + IssuerRef: cmmeta.ObjectReference{ + Name: issuerName(cr), + Kind: issuerKind, + Group: issuerGroup, + }, + }, + } +} diff --git a/pkg/psmdb/tls/certmanager.go b/pkg/psmdb/tls/certmanager.go index aa14cd746c..e53aa0f3b2 100644 --- a/pkg/psmdb/tls/certmanager.go +++ b/pkg/psmdb/tls/certmanager.go @@ -6,7 +6,6 @@ import ( "time" cm "github.com/cert-manager/cert-manager/pkg/apis/certmanager/v1" - cmmeta "github.com/cert-manager/cert-manager/pkg/apis/meta/v1" "github.com/cert-manager/cert-manager/pkg/util/cmapichecker" "github.com/pkg/errors" corev1 "k8s.io/api/core/v1" @@ -27,10 +26,9 @@ import ( type CertManagerController interface { ApplyIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) ApplyCAIssuer(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) - ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, internal bool) (util.ApplyStatus, error) - ApplyCACertificate(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) + ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, cert Certificate) (util.ApplyStatus, error) DeleteDeprecatedIssuerIfExists(ctx context.Context, cr *api.PerconaServerMongoDB) error - WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, secretsList ...string) error + WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, certificates ...Certificate) error GetMergedCA(ctx context.Context, cr *api.PerconaServerMongoDB, secretNames []string) ([]byte, error) Check(ctx context.Context, config *rest.Config, ns string) error IsDryRun() bool @@ -62,21 +60,6 @@ func (c *certManagerController) IsDryRun() bool { return c.dryRun } -func certificateName(cr *api.PerconaServerMongoDB, internal bool) string { - if internal { - return cr.Name + "-ssl-internal" - } - return cr.Name + "-ssl" -} - -func CertificateSecretName(cr *api.PerconaServerMongoDB, internal bool) string { - if internal { - return api.SSLInternalSecretName(cr) - } - - return api.SSLSecretName(cr) -} - func deprecatedIssuerName(cr *api.PerconaServerMongoDB) string { return cr.Name + "-psmdb-ca" } @@ -97,10 +80,6 @@ func caIssuerName(cr *api.PerconaServerMongoDB) string { return cr.Name + "-psmdb-ca-issuer" } -func CACertificateSecretName(cr *api.PerconaServerMongoDB) string { - return cr.Name + "-ca-cert" -} - func (c *certManagerController) DeleteDeprecatedIssuerIfExists(ctx context.Context, cr *api.PerconaServerMongoDB) error { issuer := new(cm.Issuer) err := c.cl.Get(ctx, types.NamespacedName{ @@ -138,7 +117,7 @@ func (c *certManagerController) ApplyIssuer(ctx context.Context, cr *api.Percona Spec: cm.IssuerSpec{ IssuerConfig: cm.IssuerConfig{ CA: &cm.CAIssuer{ - SecretName: CACertificateSecretName(cr), + SecretName: CertificateCA(cr).SecretName(), }, }, }, @@ -180,47 +159,8 @@ func (c *certManagerController) ApplyCAIssuer(ctx context.Context, cr *api.Perco return c.createOrUpdate(ctx, cr, issuer) } -func (c *certManagerController) ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, internal bool) (util.ApplyStatus, error) { - issuerKind := cm.IssuerKind - issuerGroup := "" - if cr.CompareVersion("1.16.0") >= 0 && cr.Spec.TLS != nil && cr.Spec.TLS.IssuerConf != nil { - issuerKind = cr.Spec.TLS.IssuerConf.Kind - issuerGroup = cr.Spec.TLS.IssuerConf.Group - - } - isCA := false - if cr.CompareVersion("1.15.0") < 0 { - isCA = true - } - - certificate := &cm.Certificate{ - ObjectMeta: metav1.ObjectMeta{ - Name: certificateName(cr, internal), - Namespace: cr.Namespace, - Labels: naming.ClusterLabels(cr), - }, - Spec: cm.CertificateSpec{ - Subject: &cm.X509Subject{ - Organizations: []string{"PSMDB"}, - }, - CommonName: cr.Name, - SecretName: CertificateSecretName(cr, internal), - DNSNames: GetCertificateSans(cr), - IsCA: isCA, - Duration: &cr.Spec.TLS.CertValidityDuration, - IssuerRef: cmmeta.ObjectReference{ - Name: issuerName(cr), - Kind: issuerKind, - Group: issuerGroup, - }, - }, - } - - if cr.CompareVersion("1.17.0") < 0 { - certificate.Labels = nil - } - - return c.createOrUpdate(ctx, cr, certificate) +func (c *certManagerController) ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, cert Certificate) (util.ApplyStatus, error) { + return c.createOrUpdate(ctx, cr, cert.Object()) } var ( @@ -260,33 +200,7 @@ func translateCheckError(err error) error { return cmapichecker.TranslateToSimpleError(err) } -func (c *certManagerController) ApplyCACertificate(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) { - cert := &cm.Certificate{ - ObjectMeta: metav1.ObjectMeta{ - Name: CACertificateSecretName(cr), - Namespace: cr.Namespace, - Labels: naming.ClusterLabels(cr), - }, - Spec: cm.CertificateSpec{ - SecretName: CACertificateSecretName(cr), - CommonName: cr.Name + "-ca", - IsCA: true, - IssuerRef: cmmeta.ObjectReference{ - Name: caIssuerName(cr), - Kind: cm.IssuerKind, - }, - Duration: &metav1.Duration{Duration: time.Hour * 24 * 365}, - RenewBefore: &metav1.Duration{Duration: 730 * time.Hour}, - }, - } - if cr.CompareVersion("1.17.0") < 0 { - cert.Labels = nil - } - - return c.createOrUpdate(ctx, cr, cert) -} - -func (c *certManagerController) WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, secretsList ...string) error { +func (c *certManagerController) WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, certificates ...Certificate) error { if c.dryRun { return nil } @@ -297,20 +211,28 @@ func (c *certManagerController) WaitForCerts(ctx context.Context, cr *api.Percon for { select { case <-timeoutTimer.C: - return errors.Errorf("timeout: can't get tls certificates from certmanager, %s", secretsList) + return errors.Errorf("timeout: can't get tls certificates from certmanager, %v", certificates) case <-ticker.C: successCount := 0 - for _, secretName := range secretsList { + for _, cert := range certificates { secret := &corev1.Secret{} err := c.cl.Get(ctx, types.NamespacedName{ - Name: secretName, + Name: cert.SecretName(), Namespace: cr.Namespace, }, secret) if err != nil && !k8serrors.IsNotFound(err) { return err } else if err == nil { successCount++ - if v, ok := secret.Annotations[cm.CertificateNameKey]; !ok || v != secret.Name { + if v, ok := secret.Annotations[cm.CertificateNameKey]; !ok || v != cert.Name() { + continue + } + certificate := &cm.Certificate{} + err := c.cl.Get(ctx, client.ObjectKeyFromObject(cert.Object()), certificate) + if err != nil { + return err + } + if metav1.IsControlledBy(secret, certificate) { continue } if err = controllerutil.SetControllerReference(cr, secret, c.scheme); err != nil { @@ -321,7 +243,7 @@ func (c *certManagerController) WaitForCerts(ctx context.Context, cr *api.Percon } } } - if successCount == len(secretsList) { + if successCount == len(certificates) { return nil } } diff --git a/pkg/psmdb/tls/certmanager_test.go b/pkg/psmdb/tls/certmanager_test.go index 18fec27ad8..1349d4f7f9 100644 --- a/pkg/psmdb/tls/certmanager_test.go +++ b/pkg/psmdb/tls/certmanager_test.go @@ -97,11 +97,12 @@ func TestCreateCertificate(t *testing.T) { cert := &cm.Certificate{} t.Run("Create certificate with custom issuer name", func(t *testing.T) { - if _, err := r.ApplyCertificate(ctx, cr, false); err != nil { + tlsCert := CertificateTLS(cr, false) + if _, err := r.ApplyCertificate(ctx, cr, tlsCert); err != nil { t.Fatal(err) } - err := r.GetClient().Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: certificateName(cr, false)}, cert) + err := r.GetClient().Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: tlsCert.Name()}, cert) if err != nil { t.Fatal(err) } @@ -115,11 +116,12 @@ func TestCreateCertificate(t *testing.T) { cr.Name = "psmdb-mock-1" cr.Spec.CRVersion = "1.15.0" - if _, err := r.ApplyCertificate(ctx, cr, false); err != nil { + tlsCert := CertificateTLS(cr, false) + if _, err := r.ApplyCertificate(ctx, cr, tlsCert); err != nil { t.Fatal(err) } - err := r.GetClient().Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: certificateName(cr, false)}, cert) + err := r.GetClient().Get(ctx, types.NamespacedName{Namespace: "psmdb", Name: tlsCert.Name()}, cert) if err != nil { t.Fatal(err) } diff --git a/pkg/psmdb/tls/fake/certmanager.go b/pkg/psmdb/tls/fake/certmanager.go index b1b59a6dbc..c3ebedbc8c 100644 --- a/pkg/psmdb/tls/fake/certmanager.go +++ b/pkg/psmdb/tls/fake/certmanager.go @@ -34,11 +34,7 @@ func (c *fakeCertManagerController) ApplyCAIssuer(ctx context.Context, cr *api.P return util.ApplyStatusUnchanged, nil } -func (c *fakeCertManagerController) ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, internal bool) (util.ApplyStatus, error) { - return util.ApplyStatusUnchanged, nil -} - -func (c *fakeCertManagerController) ApplyCACertificate(ctx context.Context, cr *api.PerconaServerMongoDB) (util.ApplyStatus, error) { +func (c *fakeCertManagerController) ApplyCertificate(ctx context.Context, cr *api.PerconaServerMongoDB, cert tls.Certificate) (util.ApplyStatus, error) { return util.ApplyStatusUnchanged, nil } @@ -46,7 +42,7 @@ func (c *fakeCertManagerController) DeleteDeprecatedIssuerIfExists(ctx context.C return nil } -func (c *fakeCertManagerController) WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, secretsList ...string) error { +func (c *fakeCertManagerController) WaitForCerts(ctx context.Context, cr *api.PerconaServerMongoDB, cert ...tls.Certificate) error { return nil }