From 08340d6791d0fad961aa21bf0a8a7ea3c676078f Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Fri, 1 Aug 2025 15:29:43 +0200 Subject: [PATCH 01/11] wip: external mongod --- api/v1/search/mongodbsearch_types.go | 15 ++ api/v1/search/zz_generated.deepcopy.go | 55 ++++++++ .../operator/mongodbsearch_controller.go | 16 ++- .../mongodbsearch_reconcile_helper.go | 18 +-- .../search_controller/search_construction.go | 58 ++++++-- .../search_community_external_mongod_basic.py | 133 ++++++++++++++++++ .../controllers/replica_set_controller.go | 3 +- 7 files changed, 276 insertions(+), 22 deletions(-) create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index f07f25053..f5a1a498b 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -45,11 +45,26 @@ type MongoDBSource struct { // +optional MongoDBResourceRef *userv1.MongoDBResourceRef `json:"mongodbResourceRef,omitempty"` // +optional + ExternalMongoDBSource *ExternalMongoDBSource `json:"external,omitempty"` + // +optional PasswordSecretRef *userv1.SecretKeyRef `json:"passwordSecretRef,omitempty"` // +optional Username *string `json:"username,omitempty"` } +type ExternalMongoDBSource struct { + HostAndPorts []string `json:"hostAndPorts,omitempty"` + KeyFileSecretKeyRef *userv1.SecretKeyRef `json:"keyFileSecretRef,omitempty"` // This is the mongod credential used to connect to the external MongoDB deployment + // +optional + TLS *ExternalMongodTLS `json:"tls,omitempty"` // TLS configuration for the external MongoDB deployment +} + +type ExternalMongodTLS struct { + Enabled bool `json:"enabled"` + // +optional + CASecretRef *userv1.SecretKeyRef `json:"caSecretRef,omitempty"` +} + type Security struct { // +optional TLS TLS `json:"tls"` diff --git a/api/v1/search/zz_generated.deepcopy.go b/api/v1/search/zz_generated.deepcopy.go index 8817e4b46..eb015e9d1 100644 --- a/api/v1/search/zz_generated.deepcopy.go +++ b/api/v1/search/zz_generated.deepcopy.go @@ -28,6 +28,56 @@ import ( runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalMongoDBSource) DeepCopyInto(out *ExternalMongoDBSource) { + *out = *in + if in.HostAndPorts != nil { + in, out := &in.HostAndPorts, &out.HostAndPorts + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.KeyFileSecretKeyRef != nil { + in, out := &in.KeyFileSecretKeyRef, &out.KeyFileSecretKeyRef + *out = new(user.SecretKeyRef) + **out = **in + } + if in.TLS != nil { + in, out := &in.TLS, &out.TLS + *out = new(ExternalMongodTLS) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalMongoDBSource. +func (in *ExternalMongoDBSource) DeepCopy() *ExternalMongoDBSource { + if in == nil { + return nil + } + out := new(ExternalMongoDBSource) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ExternalMongodTLS) DeepCopyInto(out *ExternalMongodTLS) { + *out = *in + if in.CASecretRef != nil { + in, out := &in.CASecretRef, &out.CASecretRef + *out = new(user.SecretKeyRef) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExternalMongodTLS. +func (in *ExternalMongodTLS) DeepCopy() *ExternalMongodTLS { + if in == nil { + return nil + } + out := new(ExternalMongodTLS) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *MongoDBSearch) DeepCopyInto(out *MongoDBSearch) { *out = *in @@ -151,6 +201,11 @@ func (in *MongoDBSource) DeepCopyInto(out *MongoDBSource) { *out = new(user.MongoDBResourceRef) **out = **in } + if in.ExternalMongoDBSource != nil { + in, out := &in.ExternalMongoDBSource, &out.ExternalMongoDBSource + *out = new(ExternalMongoDBSource) + (*in).DeepCopyInto(*out) + } if in.PasswordSecretRef != nil { in, out := &in.PasswordSecretRef, &out.PasswordSecretRef *out = new(user.SecretKeyRef) diff --git a/controllers/operator/mongodbsearch_controller.go b/controllers/operator/mongodbsearch_controller.go index bf8a5f022..7c41d65f0 100644 --- a/controllers/operator/mongodbsearch_controller.go +++ b/controllers/operator/mongodbsearch_controller.go @@ -51,26 +51,32 @@ func (r *MongoDBSearchReconciler) Reconcile(ctx context.Context, request reconci return result, err } - sourceResource, err := getSourceMongoDBForSearch(ctx, r.kubeClient, mdbSearch) + sourceResource, mdbc, err := getSourceMongoDBForSearch(ctx, r.kubeClient, mdbSearch) if err != nil { return reconcile.Result{RequeueAfter: time.Second * util.RetryTimeSec}, err } - r.mdbcWatcher.Watch(ctx, sourceResource.NamespacedName(), request.NamespacedName) + if mdbc != nil { + r.mdbcWatcher.Watch(ctx, mdbc.NamespacedName(), request.NamespacedName) + } reconcileHelper := search_controller.NewMongoDBSearchReconcileHelper(kubernetesClient.NewClient(r.kubeClient), mdbSearch, sourceResource, r.operatorSearchConfig) return reconcileHelper.Reconcile(ctx, log).ReconcileResult() } -func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch) (search_controller.SearchSourceDBResource, error) { +func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch) (search_controller.SearchSourceDBResource, *mdbcv1.MongoDBCommunity, error) { + if search.Spec.Source != nil && search.Spec.Source.ExternalMongoDBSource != nil { + return search_controller.NewSearchSourceDBResourceFromExternal(search.Namespace, search.Spec.Source.ExternalMongoDBSource), nil, nil + } + sourceMongoDBResourceRef := search.GetMongoDBResourceRef() mdbcName := types.NamespacedName{Namespace: search.GetNamespace(), Name: sourceMongoDBResourceRef.Name} mdbc := &mdbcv1.MongoDBCommunity{} if err := kubeClient.Get(ctx, mdbcName, mdbc); err != nil { - return nil, xerrors.Errorf("error getting MongoDBCommunity %s: %w", mdbcName, err) + return nil, nil, xerrors.Errorf("error getting MongoDBCommunity %s: %w", mdbcName, err) } - return search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), nil + return search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), nil, nil } func mdbcSearchIndexBuilder(rawObj client.Object) []string { diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper.go b/controllers/search_controller/mongodbsearch_reconcile_helper.go index 08a264853..f0b930d01 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper.go @@ -123,7 +123,7 @@ func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.S return workflow.Failed(err) } - if statefulSetStatus := statefulset.GetStatefulSetStatus(ctx, r.db.NamespacedName().Namespace, r.mdbSearch.StatefulSetNamespacedName().Name, r.client); !statefulSetStatus.IsOK() { + if statefulSetStatus := statefulset.GetStatefulSetStatus(ctx, r.mdbSearch.Namespace, r.mdbSearch.StatefulSetNamespacedName().Name, r.client); !statefulSetStatus.IsOK() { return statefulSetStatus } @@ -334,10 +334,7 @@ func buildSearchHeadlessService(search *searchv1.MongoDBSearch) corev1.Service { func createMongotConfig(search *searchv1.MongoDBSearch, db SearchSourceDBResource) mongot.Modification { return func(config *mongot.Config) { - var hostAndPorts []string - for i := range db.Members() { - hostAndPorts = append(hostAndPorts, fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", db.Name(), i, db.DatabaseServiceName(), db.GetNamespace(), db.DatabasePort())) - } + hostAndPorts := db.HostSeeds() config.SyncSource = mongot.ConfigSyncSource{ ReplicaSet: mongot.ConfigReplicaSet{ @@ -407,11 +404,16 @@ func ValidateSearchSource(db SearchSourceDBResource) error { } func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSource(ctx context.Context) error { + if r.mdbSearch.Spec.Source != nil && r.mdbSearch.Spec.Source.ExternalMongoDBSource != nil { + return nil + } + + ref := r.mdbSearch.GetMongoDBResourceRef() searchList := &searchv1.MongoDBSearchList{} if err := r.client.List(ctx, searchList, &client.ListOptions{ - FieldSelector: fields.OneTermEqualSelector(MongoDBSearchIndexFieldName, r.db.GetNamespace()+"/"+r.db.Name()), + FieldSelector: fields.OneTermEqualSelector(MongoDBSearchIndexFieldName, ref.Namespace+"/"+ref.Name), }); err != nil { - return xerrors.Errorf("Error listing MongoDBSearch resources for search source '%s': %w", r.db.Name(), err) + return xerrors.Errorf("Error listing MongoDBSearch resources for search source '%s': %w", ref.Name, err) } if len(searchList.Items) > 1 { @@ -420,7 +422,7 @@ func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSourc resourceNames[i] = search.Name } return xerrors.Errorf( - "Found multiple MongoDBSearch resources for search source '%s': %s", r.db.Name(), + "Found multiple MongoDBSearch resources for search source '%s': %s", ref.Name, strings.Join(resourceNames, ", "), ) } diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index 6aae8e982..751e51acf 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -1,6 +1,7 @@ package search_controller import ( + "fmt" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" @@ -31,27 +32,67 @@ const ( // // TODO check if we could use already existing interface (DbCommon, MongoDBStatefulSetOwner, etc.) type SearchSourceDBResource interface { - Name() string - NamespacedName() types.NamespacedName KeyfileSecretName() string - GetNamespace() string - HasSeparateDataAndLogsVolumes() bool - DatabaseServiceName() string - DatabasePort() int GetMongoDBVersion() string IsSecurityTLSConfigEnabled() bool TLSOperatorCASecretNamespacedName() types.NamespacedName - Members() int + HostSeeds() []string } func NewSearchSourceDBResourceFromMongoDBCommunity(mdbc *mdbcv1.MongoDBCommunity) SearchSourceDBResource { return &mdbcSearchResource{db: mdbc} } +func NewSearchSourceDBResourceFromExternal(namespace string, spec *searchv1.ExternalMongoDBSource) SearchSourceDBResource { + return &externalSearchResource{namespace: namespace, spec: spec} +} + +// externalSearchResource implements SearchSourceDBResource for deployments managed outside the operator. +type externalSearchResource struct { + namespace string + spec *searchv1.ExternalMongoDBSource +} + +func (r *externalSearchResource) KeyfileSecretName() string { + if r.spec.KeyFileSecretKeyRef != nil { + return r.spec.KeyFileSecretKeyRef.Name + } + + return "" +} + +func (r *externalSearchResource) GetMongoDBVersion() string { + return "8.0.10" // replace this with a validate method that is always true for external mongodb +} + +func (r *externalSearchResource) IsSecurityTLSConfigEnabled() bool { + if r.spec.TLS != nil { + return r.spec.TLS.Enabled + } + return false +} + +func (r *externalSearchResource) TLSOperatorCASecretNamespacedName() types.NamespacedName { + if r.spec.TLS != nil { + return types.NamespacedName{Name: r.spec.TLS.CASecretRef.Name, Namespace: r.namespace} + } + return types.NamespacedName{} +} + +func (r *externalSearchResource) HostSeeds() []string { return r.spec.HostAndPorts } + type mdbcSearchResource struct { db *mdbcv1.MongoDBCommunity } +func (r *mdbcSearchResource) HostSeeds() []string { + seeds := make([]string, r.db.Spec.Members) + for i := range seeds { + seeds[i] = fmt.Sprintf("%s-%d.%s.%s.svc.cluster.local:%d", r.db.Name, i, r.db.ServiceName(), r.db.Namespace, r.db.GetMongodConfiguration().GetDBPort()) + } + return seeds +} + func (r *mdbcSearchResource) Members() int { return r.db.Spec.Members } @@ -80,6 +121,7 @@ func (r *mdbcSearchResource) DatabaseServiceName() string { return r.db.ServiceName() } +// replace with a validate method that is always true for external mongodb func (r *mdbcSearchResource) GetMongoDBVersion() string { return r.db.Spec.Version } @@ -161,7 +203,7 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso podSecurityContext, podtemplatespec.WithPodLabels(labels), podtemplatespec.WithVolumes(volumes), - podtemplatespec.WithServiceAccount(sourceDBResource.DatabaseServiceName()), + //podtemplatespec.WithServiceAccount(sourceDBResource.DatabaseServiceName()), podtemplatespec.WithServiceAccount(util.MongoDBServiceAccount), podtemplatespec.WithContainer(MongotContainerName, mongodbSearchContainer(mdbSearch, volumeMounts, searchImage)), ), diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py new file mode 100644 index 000000000..e61c4cdb1 --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -0,0 +1,133 @@ +from kubetester import create_or_update_secret, try_load +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb_community import MongoDBCommunity +from kubetester.mongodb_search import MongoDBSearch +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "mongot-user" +MONGOT_USER_PASSWORD = "mongot-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + +MDBC_RESOURCE_NAME = "mdbc-rs" + + +@fixture(scope="function") +def mdbc(namespace: str) -> MongoDBCommunity: + resource = MongoDBCommunity.from_yaml( + yaml_fixture("community-replicaset-sample-mflix.yaml"), + name=MDBC_RESOURCE_NAME, + namespace=namespace, + ) + + # if try_load(resource): + # return resource + + mongot_host = f"{MDBC_RESOURCE_NAME}-search-svc.{namespace}.svc.cluster.local:27027" + resource.setdefault("spec", {}).setdefault("additionalMongodConfig", {}).setdefault("setParameter", {}).update( + { + "mongotHost": mongot_host, + "searchIndexManagementHostAndPort": mongot_host, + "skipAuthenticationToSearchIndexManagementServer": False, + "searchTLSMode": "disabled", + } + ) + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml( + yaml_fixture("search-minimal.yaml"), + namespace=namespace, + ) + + seeds = [ + f"{mdbc.name}-{i}.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017" + for i in range(mdbc["spec"]["members"]) + ] + resource.setdefault("spec", {})["source"] = { + "external": { + "hostAndPorts": seeds, + "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile"}, + } + } + return resource + + +@mark.e2e_search_external_basic +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_external_basic +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, name=f"{mdbs.name}-{MONGOT_USER_NAME}-password", data={"password": MONGOT_USER_PASSWORD} + ) + + +@mark.e2e_search_external_basic +def test_create_database_resource(mdbc: MongoDBCommunity): + mdbc.update() + mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_external_basic +def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_external_basic +def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): + mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@fixture(scope="function") +def sample_movies_helper(mdbc: MongoDBCommunity) -> SampleMoviesSearchHelper: + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester(get_connection_string(mdbc, USER_NAME, USER_PASSWORD)) + ) + + +@mark.e2e_search_external_basic +def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_external_basic +def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.create_search_index() + + +# @mark.e2e_search_external_basic +# def test_search_wait_for_search_indexes(sample_movies_helper: SampleMoviesSearchHelper): +# sample_movies_helper.wait_for_search_indexes() + + +@mark.e2e_search_external_basic +def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}" diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index 644719294..3c6cfb5ae 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -834,7 +834,8 @@ func getMongodConfigModification(mdb mdbv1.MongoDBCommunity) automationconfig.Mo // getMongodConfigModification will merge the additional configuration in the CRD // into the configuration set up by the operator. func getMongodConfigSearchModification(search *searchv1.MongoDBSearch) automationconfig.Modification { - if search == nil { + // Condition for skipping add parameter if it is external mongod + if search == nil || search.Spec.Source.ExternalMongoDBSource != nil { return automationconfig.NOOP() } From e0b4859e29eaae6bb1d5784f6a1a309f46f4f095 Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Tue, 5 Aug 2025 11:28:38 +0200 Subject: [PATCH 02/11] tests pass --- .evergreen-tasks.yml | 5 ++++ .evergreen.yml | 1 + api/v1/search/mongodbsearch_types.go | 12 +++++++-- .../operator/mongodbsearch_controller.go | 9 +++++++ .../search_controller/search_construction.go | 1 - .../search_community_external_mongod_basic.py | 26 ++++++++++++------- .../controllers/replica_set_controller.go | 5 +++- 7 files changed, 46 insertions(+), 13 deletions(-) diff --git a/.evergreen-tasks.yml b/.evergreen-tasks.yml index 1d289fa3f..df2b2793a 100644 --- a/.evergreen-tasks.yml +++ b/.evergreen-tasks.yml @@ -1290,3 +1290,8 @@ tasks: tags: ["patch-run"] commands: - func: "e2e_test" + + - name: e2e_search_external_basic + tags: [ "patch-run" ] + commands: + - func: "e2e_test" diff --git a/.evergreen.yml b/.evergreen.yml index 94bbd5a09..dde7e1ddf 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -687,6 +687,7 @@ task_groups: - e2e_community_replicaset_scale - e2e_search_community_basic - e2e_search_community_tls + - e2e_search_external_basic # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant - name: e2e_mdb_kind_cloudqa_task_group diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index f5a1a498b..548115c93 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -178,13 +178,17 @@ func (s *MongoDBSearch) GetOwnerReferences() []metav1.OwnerReference { return []metav1.OwnerReference{ownerReference} } -func (s *MongoDBSearch) GetMongoDBResourceRef() userv1.MongoDBResourceRef { +func (s *MongoDBSearch) GetMongoDBResourceRef() *userv1.MongoDBResourceRef { + if s.IsExternalMongoDBSource() { + return nil + } + mdbResourceRef := userv1.MongoDBResourceRef{Namespace: s.Namespace, Name: s.Name} if s.Spec.Source != nil && s.Spec.Source.MongoDBResourceRef != nil && s.Spec.Source.MongoDBResourceRef.Name != "" { mdbResourceRef.Name = s.Spec.Source.MongoDBResourceRef.Name } - return mdbResourceRef + return &mdbResourceRef } func (s *MongoDBSearch) GetMongotPort() int32 { @@ -209,3 +213,7 @@ func (s *MongoDBSearch) TLSOperatorSecretNamespacedName() types.NamespacedName { func (s *MongoDBSearch) GetMongotHealthCheckPort() int32 { return MongotDefautHealthCheckPort } + +func (s *MongoDBSearch) IsExternalMongoDBSource() bool { + return s.Spec.Source != nil && s.Spec.Source.ExternalMongoDBSource != nil +} diff --git a/controllers/operator/mongodbsearch_controller.go b/controllers/operator/mongodbsearch_controller.go index 7c41d65f0..1c17bde3c 100644 --- a/controllers/operator/mongodbsearch_controller.go +++ b/controllers/operator/mongodbsearch_controller.go @@ -71,6 +71,10 @@ func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, se } sourceMongoDBResourceRef := search.GetMongoDBResourceRef() + if sourceMongoDBResourceRef == nil { + return nil, nil, xerrors.New("MongoDBSearch source MongoDB resource reference is not set") + } + mdbcName := types.NamespacedName{Namespace: search.GetNamespace(), Name: sourceMongoDBResourceRef.Name} mdbc := &mdbcv1.MongoDBCommunity{} if err := kubeClient.Get(ctx, mdbcName, mdbc); err != nil { @@ -81,6 +85,11 @@ func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, se func mdbcSearchIndexBuilder(rawObj client.Object) []string { mdbSearch := rawObj.(*searchv1.MongoDBSearch) + resourceRef := mdbSearch.GetMongoDBResourceRef() + if resourceRef == nil { + return []string{} + } + return []string{mdbSearch.GetMongoDBResourceRef().Namespace + "/" + mdbSearch.GetMongoDBResourceRef().Name} } diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index 751e51acf..b83f0cb1a 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -203,7 +203,6 @@ func CreateSearchStatefulSetFunc(mdbSearch *searchv1.MongoDBSearch, sourceDBReso podSecurityContext, podtemplatespec.WithPodLabels(labels), podtemplatespec.WithVolumes(volumes), - //podtemplatespec.WithServiceAccount(sourceDBResource.DatabaseServiceName()), podtemplatespec.WithServiceAccount(util.MongoDBServiceAccount), podtemplatespec.WithContainer(MongotContainerName, mongodbSearchContainer(mdbSearch, volumeMounts, searchImage)), ), diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py index e61c4cdb1..fe170ef9f 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -36,14 +36,18 @@ def mdbc(namespace: str) -> MongoDBCommunity: # return resource mongot_host = f"{MDBC_RESOURCE_NAME}-search-svc.{namespace}.svc.cluster.local:27027" - resource.setdefault("spec", {}).setdefault("additionalMongodConfig", {}).setdefault("setParameter", {}).update( - { - "mongotHost": mongot_host, - "searchIndexManagementHostAndPort": mongot_host, - "skipAuthenticationToSearchIndexManagementServer": False, - "searchTLSMode": "disabled", - } - ) + if "additionalMongodConfig" not in resource["spec"]: + resource["spec"]["additionalMongodConfig"] = {} + if "setParameter" not in resource["spec"]["additionalMongodConfig"]: + resource["spec"]["additionalMongodConfig"]["setParameter"] = {} + + # Update the setParameter section + resource["spec"]["additionalMongodConfig"]["setParameter"].update({ + "mongotHost": mongot_host, + "searchIndexManagementHostAndPort": mongot_host, + "skipAuthenticationToSearchIndexManagementServer": False, + "searchTLSMode": "disabled", + }) return resource @@ -59,7 +63,11 @@ def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: f"{mdbc.name}-{i}.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017" for i in range(mdbc["spec"]["members"]) ] - resource.setdefault("spec", {})["source"] = { + + if "source" not in resource["spec"]: + resource["spec"]["source"] = {} + + resource["spec"]["source"] = { "external": { "hostAndPorts": seeds, "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile"}, diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index 3c6cfb5ae..d974d1862 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -89,6 +89,9 @@ func NewReconciler(mgr manager.Manager, mongodbRepoUrl, mongodbImage, mongodbIma func findMdbcForSearch(ctx context.Context, rawObj k8sClient.Object) []reconcile.Request { mdbSearch := rawObj.(*searchv1.MongoDBSearch) + if mdbSearch.GetMongoDBResourceRef() == nil { + return nil //maybe empty array + } return []reconcile.Request{ {NamespacedName: types.NamespacedName{Namespace: mdbSearch.GetMongoDBResourceRef().Namespace, Name: mdbSearch.GetMongoDBResourceRef().Name}}, } @@ -835,7 +838,7 @@ func getMongodConfigModification(mdb mdbv1.MongoDBCommunity) automationconfig.Mo // into the configuration set up by the operator. func getMongodConfigSearchModification(search *searchv1.MongoDBSearch) automationconfig.Modification { // Condition for skipping add parameter if it is external mongod - if search == nil || search.Spec.Source.ExternalMongoDBSource != nil { + if search == nil || search.IsExternalMongoDBSource() { return automationconfig.NOOP() } From b4637f393b82fde70e8e8c221698be9930b38fab Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Tue, 5 Aug 2025 15:23:38 +0200 Subject: [PATCH 03/11] fix lint and unit test --- .../operator/mongodbsearch_controller.go | 6 +++--- .../search_controller/search_construction.go | 1 + .../search_community_external_mongod_basic.py | 17 +++++++++-------- .../controllers/replica_set_controller.go | 2 +- 4 files changed, 14 insertions(+), 12 deletions(-) diff --git a/controllers/operator/mongodbsearch_controller.go b/controllers/operator/mongodbsearch_controller.go index 1c17bde3c..b83791f1f 100644 --- a/controllers/operator/mongodbsearch_controller.go +++ b/controllers/operator/mongodbsearch_controller.go @@ -66,7 +66,7 @@ func (r *MongoDBSearchReconciler) Reconcile(ctx context.Context, request reconci } func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, search *searchv1.MongoDBSearch) (search_controller.SearchSourceDBResource, *mdbcv1.MongoDBCommunity, error) { - if search.Spec.Source != nil && search.Spec.Source.ExternalMongoDBSource != nil { + if search.IsExternalMongoDBSource() { return search_controller.NewSearchSourceDBResourceFromExternal(search.Namespace, search.Spec.Source.ExternalMongoDBSource), nil, nil } @@ -80,7 +80,7 @@ func getSourceMongoDBForSearch(ctx context.Context, kubeClient client.Client, se if err := kubeClient.Get(ctx, mdbcName, mdbc); err != nil { return nil, nil, xerrors.Errorf("error getting MongoDBCommunity %s: %w", mdbcName, err) } - return search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), nil, nil + return search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(mdbc), mdbc, nil } func mdbcSearchIndexBuilder(rawObj client.Object) []string { @@ -90,7 +90,7 @@ func mdbcSearchIndexBuilder(rawObj client.Object) []string { return []string{} } - return []string{mdbSearch.GetMongoDBResourceRef().Namespace + "/" + mdbSearch.GetMongoDBResourceRef().Name} + return []string{resourceRef.Namespace + "/" + resourceRef.Name} } func AddMongoDBSearchController(ctx context.Context, mgr manager.Manager, operatorSearchConfig search_controller.OperatorSearchConfig) error { diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index b83f0cb1a..156a3e1e0 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -2,6 +2,7 @@ package search_controller import ( "fmt" + "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py index fe170ef9f..6f3ede064 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -42,12 +42,14 @@ def mdbc(namespace: str) -> MongoDBCommunity: resource["spec"]["additionalMongodConfig"]["setParameter"] = {} # Update the setParameter section - resource["spec"]["additionalMongodConfig"]["setParameter"].update({ - "mongotHost": mongot_host, - "searchIndexManagementHostAndPort": mongot_host, - "skipAuthenticationToSearchIndexManagementServer": False, - "searchTLSMode": "disabled", - }) + resource["spec"]["additionalMongodConfig"]["setParameter"].update( + { + "mongotHost": mongot_host, + "searchIndexManagementHostAndPort": mongot_host, + "skipAuthenticationToSearchIndexManagementServer": False, + "searchTLSMode": "disabled", + } + ) return resource @@ -60,8 +62,7 @@ def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: ) seeds = [ - f"{mdbc.name}-{i}.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017" - for i in range(mdbc["spec"]["members"]) + f"{mdbc.name}-{i}.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017" for i in range(mdbc["spec"]["members"]) ] if "source" not in resource["spec"]: diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index d974d1862..5052974d6 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -90,7 +90,7 @@ func NewReconciler(mgr manager.Manager, mongodbRepoUrl, mongodbImage, mongodbIma func findMdbcForSearch(ctx context.Context, rawObj k8sClient.Object) []reconcile.Request { mdbSearch := rawObj.(*searchv1.MongoDBSearch) if mdbSearch.GetMongoDBResourceRef() == nil { - return nil //maybe empty array + return nil // maybe empty array } return []reconcile.Request{ {NamespacedName: types.NamespacedName{Namespace: mdbSearch.GetMongoDBResourceRef().Namespace, Name: mdbSearch.GetMongoDBResourceRef().Name}}, From 6938b0c845ba620b322db6b66668ec875f541d66 Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Tue, 5 Aug 2025 16:51:14 +0200 Subject: [PATCH 04/11] refactor mongodb version validation --- .../mongodbsearch_reconcile_helper.go | 14 +--------- .../mongodbsearch_reconcile_helper_test.go | 2 +- .../search_controller/search_construction.go | 26 +++++++++++++------ .../controllers/replica_set_controller.go | 2 +- 4 files changed, 21 insertions(+), 23 deletions(-) diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper.go b/controllers/search_controller/mongodbsearch_reconcile_helper.go index f0b930d01..1b0f33603 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper.go @@ -7,7 +7,6 @@ import ( "fmt" "strings" - "github.com/blang/semver" "github.com/ghodss/yaml" "go.uber.org/zap" "golang.org/x/xerrors" @@ -82,7 +81,7 @@ func (r *MongoDBSearchReconcileHelper) reconcile(ctx context.Context, log *zap.S log = log.With("MongoDBSearch", r.mdbSearch.NamespacedName()) log.Infof("Reconciling MongoDBSearch") - if err := ValidateSearchSource(r.db); err != nil { + if err := r.db.ValidateMongoDBVersion(); err != nil { return workflow.Failed(err) } @@ -392,17 +391,6 @@ func mongotHostAndPort(search *searchv1.MongoDBSearch) string { return fmt.Sprintf("%s.%s.svc.cluster.local:%d", svcName.Name, svcName.Namespace, search.GetMongotPort()) } -func ValidateSearchSource(db SearchSourceDBResource) error { - version, err := semver.ParseTolerant(db.GetMongoDBVersion()) - if err != nil { - return xerrors.Errorf("error parsing MongoDB version '%s': %w", db.GetMongoDBVersion(), err) - } else if version.LT(semver.MustParse("8.0.10")) { - return xerrors.New("MongoDB version must be 8.0.10 or higher") - } - - return nil -} - func (r *MongoDBSearchReconcileHelper) ValidateSingleMongoDBSearchForSearchSource(ctx context.Context) error { if r.mdbSearch.Spec.Source != nil && r.mdbSearch.Spec.Source.ExternalMongoDBSource != nil { return nil diff --git a/controllers/search_controller/mongodbsearch_reconcile_helper_test.go b/controllers/search_controller/mongodbsearch_reconcile_helper_test.go index f70ec9a03..c7b4a8dc7 100644 --- a/controllers/search_controller/mongodbsearch_reconcile_helper_test.go +++ b/controllers/search_controller/mongodbsearch_reconcile_helper_test.go @@ -64,7 +64,7 @@ func TestMongoDBSearchReconcileHelper_ValidateSearchSource(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { db := NewSearchSourceDBResourceFromMongoDBCommunity(&c.mdbc) - err := ValidateSearchSource(db) + err := db.ValidateMongoDBVersion() if c.expectedError == "" { assert.NoError(t, err) } else { diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index 156a3e1e0..81e756368 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -9,6 +9,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "github.com/blang/semver" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" "github.com/mongodb/mongodb-kubernetes/controllers/operator/construct" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" @@ -18,6 +19,7 @@ import ( "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/probes" "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" "github.com/mongodb/mongodb-kubernetes/pkg/util" + "golang.org/x/xerrors" ) const ( @@ -34,10 +36,10 @@ const ( // TODO check if we could use already existing interface (DbCommon, MongoDBStatefulSetOwner, etc.) type SearchSourceDBResource interface { KeyfileSecretName() string - GetMongoDBVersion() string IsSecurityTLSConfigEnabled() bool TLSOperatorCASecretNamespacedName() types.NamespacedName HostSeeds() []string + ValidateMongoDBVersion() error } func NewSearchSourceDBResourceFromMongoDBCommunity(mdbc *mdbcv1.MongoDBCommunity) SearchSourceDBResource { @@ -54,6 +56,10 @@ type externalSearchResource struct { spec *searchv1.ExternalMongoDBSource } +func (r *externalSearchResource) ValidateMongoDBVersion() error { + return nil +} + func (r *externalSearchResource) KeyfileSecretName() string { if r.spec.KeyFileSecretKeyRef != nil { return r.spec.KeyFileSecretKeyRef.Name @@ -62,10 +68,6 @@ func (r *externalSearchResource) KeyfileSecretName() string { return "" } -func (r *externalSearchResource) GetMongoDBVersion() string { - return "8.0.10" // replace this with a validate method that is always true for external mongodb -} - func (r *externalSearchResource) IsSecurityTLSConfigEnabled() bool { if r.spec.TLS != nil { return r.spec.TLS.Enabled @@ -86,6 +88,17 @@ type mdbcSearchResource struct { db *mdbcv1.MongoDBCommunity } +func (r *mdbcSearchResource) ValidateMongoDBVersion() error { + version, err := semver.ParseTolerant(r.db.GetMongoDBVersion()) + if err != nil { + return xerrors.Errorf("error parsing MongoDB version '%s': %w", r.db.GetMongoDBVersion(), err) + } else if version.LT(semver.MustParse("8.0.10")) { + return xerrors.New("MongoDB version must be 8.0.10 or higher") + } + + return nil +} + func (r *mdbcSearchResource) HostSeeds() []string { seeds := make([]string, r.db.Spec.Members) for i := range seeds { @@ -123,9 +136,6 @@ func (r *mdbcSearchResource) DatabaseServiceName() string { } // replace with a validate method that is always true for external mongodb -func (r *mdbcSearchResource) GetMongoDBVersion() string { - return r.db.Spec.Version -} func (r *mdbcSearchResource) IsSecurityTLSConfigEnabled() bool { return r.db.Spec.Security.TLS.Enabled diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index 5052974d6..52bbcb938 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -716,7 +716,7 @@ func (r ReplicaSetReconciler) buildAutomationConfig(ctx context.Context, mdb mdb // for the mongod automation config. if len(searchList.Items) == 1 { searchSource := search_controller.NewSearchSourceDBResourceFromMongoDBCommunity(&mdb) - if search_controller.ValidateSearchSource(searchSource) == nil { + if searchSource.ValidateMongoDBVersion() == nil { search = &searchList.Items[0] } } From 6c825e426c16ff0302b6889d11824bb701868e3d Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Tue, 5 Aug 2025 18:38:17 +0200 Subject: [PATCH 05/11] fix mongot username --- .../tests/search/search_community_external_mongod_basic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py index 6f3ede064..f2298c069 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -15,8 +15,8 @@ ADMIN_USER_NAME = "mdb-admin-user" ADMIN_USER_PASSWORD = "mdb-admin-user-pass" -MONGOT_USER_NAME = "mongot-user" -MONGOT_USER_PASSWORD = "mongot-user-password" +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-password" USER_NAME = "mdb-user" USER_PASSWORD = "mdb-user-pass" From d6261f80df731a7c65bee4e496a0023976b63e93 Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Tue, 12 Aug 2025 14:26:33 +0200 Subject: [PATCH 06/11] fix lint --- controllers/search_controller/search_construction.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index 81e756368..f9635c99e 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -3,13 +3,13 @@ package search_controller import ( "fmt" - "k8s.io/apimachinery/pkg/types" - "k8s.io/apimachinery/pkg/util/intstr" - + "github.com/blang/semver" + "golang.org/x/xerrors" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" - "github.com/blang/semver" searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" "github.com/mongodb/mongodb-kubernetes/controllers/operator/construct" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" @@ -19,7 +19,6 @@ import ( "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/pkg/kube/probes" "github.com/mongodb/mongodb-kubernetes/pkg/statefulset" "github.com/mongodb/mongodb-kubernetes/pkg/util" - "golang.org/x/xerrors" ) const ( From 74d4708c1fcb9e5a940980c0dc00a16e5d953514 Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Tue, 12 Aug 2025 16:22:13 +0200 Subject: [PATCH 07/11] fix lint --- controllers/search_controller/search_construction.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index f9635c99e..fe6bbbc64 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -5,11 +5,12 @@ import ( "github.com/blang/semver" "golang.org/x/xerrors" - appsv1 "k8s.io/api/apps/v1" - corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + searchv1 "github.com/mongodb/mongodb-kubernetes/api/v1/search" "github.com/mongodb/mongodb-kubernetes/controllers/operator/construct" mdbcv1 "github.com/mongodb/mongodb-kubernetes/mongodb-community-operator/api/v1" From 5b13320c08a72040dbfae52a2fae24de291762ca Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Wed, 13 Aug 2025 10:05:01 +0200 Subject: [PATCH 08/11] add tls test --- .evergreen-tasks.yml | 5 + .evergreen.yml | 1 + .../search_community_external_mongod_tls.py | 207 ++++++++++++++++++ 3 files changed, 213 insertions(+) create mode 100644 docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py diff --git a/.evergreen-tasks.yml b/.evergreen-tasks.yml index df2b2793a..01305de88 100644 --- a/.evergreen-tasks.yml +++ b/.evergreen-tasks.yml @@ -1295,3 +1295,8 @@ tasks: tags: [ "patch-run" ] commands: - func: "e2e_test" + + - name: e2e_search_external_tls + tags: [ "patch-run" ] + commands: + - func: "e2e_test" diff --git a/.evergreen.yml b/.evergreen.yml index dde7e1ddf..b23e8c767 100644 --- a/.evergreen.yml +++ b/.evergreen.yml @@ -688,6 +688,7 @@ task_groups: - e2e_search_community_basic - e2e_search_community_tls - e2e_search_external_basic + - e2e_search_external_tls # This is the task group that contains all the tests run in the e2e_mdb_kind_ubuntu_cloudqa build variant - name: e2e_mdb_kind_cloudqa_task_group diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py new file mode 100644 index 000000000..edb8a83dd --- /dev/null +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py @@ -0,0 +1,207 @@ +import pymongo +from kubetester import create_or_update_secret, try_load +from kubetester.certs import create_tls_certs +from kubetester.kubetester import fixture as yaml_fixture +from kubetester.mongodb_community import MongoDBCommunity +from kubetester.mongodb_search import MongoDBSearch +from kubetester.phase import Phase +from pytest import fixture, mark +from tests import test_logger +from tests.common.search import movies_search_helper +from tests.common.search.movies_search_helper import SampleMoviesSearchHelper +from tests.common.search.search_tester import SearchTester +from tests.conftest import get_default_operator + +logger = test_logger.get_test_logger(__name__) + +ADMIN_USER_NAME = "mdb-admin-user" +ADMIN_USER_PASSWORD = "mdb-admin-user-pass" + +MONGOT_USER_NAME = "search-sync-source" +MONGOT_USER_PASSWORD = "search-sync-source-user-password" + +USER_NAME = "mdb-user" +USER_PASSWORD = "mdb-user-pass" + +MDBC_RESOURCE_NAME = "mdbc-rs" +TLS_SECRET_NAME = "tls-secret" +TLS_CA_SECRET_NAME = "tls-ca-secret" +MDBS_TLS_SECRET_NAME = "mdbs-tls-secret" + + +@fixture(scope="function") +def mdbc(namespace: str) -> MongoDBCommunity: + resource = MongoDBCommunity.from_yaml( + yaml_fixture("community-replicaset-sample-mflix.yaml"), + name=MDBC_RESOURCE_NAME, + namespace=namespace, + ) + + if try_load(resource): + return resource + + mongot_host = f"{MDBC_RESOURCE_NAME}-search-svc.{namespace}.svc.cluster.local:27027" + if "additionalMongodConfig" not in resource["spec"]: + resource["spec"]["additionalMongodConfig"] = {} + if "setParameter" not in resource["spec"]["additionalMongodConfig"]: + resource["spec"]["additionalMongodConfig"]["setParameter"] = {} + + resource["spec"]["additionalMongodConfig"]["setParameter"].update( + { + "mongotHost": mongot_host, + "searchIndexManagementHostAndPort": mongot_host, + "skipAuthenticationToSearchIndexManagementServer": False, + } + ) + + resource["spec"]["security"]["tls"] = { + "enabled": True, + "certificateKeySecretRef": {"name": TLS_SECRET_NAME}, + "caCertificateSecretRef": {"name": TLS_SECRET_NAME}, + } + + return resource + + +@fixture(scope="function") +def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: + resource = MongoDBSearch.from_yaml( + yaml_fixture("search-minimal.yaml"), + namespace=namespace, + ) + + if try_load(resource): + return resource + + seeds = [ + f"{mdbc.name}-{i}.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017" for i in range(mdbc["spec"]["members"]) + ] + + resource["spec"]["source"] = { + "external": { + "hostAndPorts": seeds, + "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile"}, + "tls": { + "enabled": True, + "caSecretRef": {"name": TLS_CA_SECRET_NAME, "key": "ca.crt"}, + }, + } + } + + if "spec" not in resource: + resource["spec"] = {} + resource["spec"]["security"] = {"tls": {"enabled": True, "certificateKeySecretRef": {"name": MDBS_TLS_SECRET_NAME}}} + + return resource + + +@mark.e2e_search_external_tls +def test_install_operator(namespace: str, operator_installation_config: dict[str, str]): + operator = get_default_operator(namespace, operator_installation_config=operator_installation_config) + operator.assert_is_running() + + +@mark.e2e_search_external_tls +def test_install_secrets(namespace: str, mdbs: MongoDBSearch): + create_or_update_secret(namespace=namespace, name=f"{USER_NAME}-password", data={"password": USER_PASSWORD}) + create_or_update_secret( + namespace=namespace, name=f"{ADMIN_USER_NAME}-password", data={"password": ADMIN_USER_PASSWORD} + ) + create_or_update_secret( + namespace=namespace, name=f"{mdbs.name}-{MONGOT_USER_NAME}-password", data={"password": MONGOT_USER_PASSWORD} + ) + + +@mark.e2e_search_external_tls +def test_install_tls_secrets_and_configmaps(namespace: str, mdbc: MongoDBCommunity, mdbs: MongoDBSearch, issuer: str, + issuer_ca_filepath: str): + create_tls_certs(issuer, namespace, mdbc.name, mdbc["spec"]["members"], secret_name=TLS_SECRET_NAME) + + search_service_name = f"{mdbs.name}-search-svc" + create_tls_certs( + issuer, + namespace, + f"{mdbs.name}-search", + replicas=1, + service_name=search_service_name, + additional_domains=[f"{search_service_name}.{namespace}.svc.cluster.local"], + secret_name=MDBS_TLS_SECRET_NAME, + ) + + ca = open(issuer_ca_filepath).read() + create_or_update_secret(namespace=namespace, name=TLS_CA_SECRET_NAME, data={"ca.crt": ca}) + + +@mark.e2e_search_external_tls +def test_create_database_resource(mdbc: MongoDBCommunity): + mdbc.update() + mdbc.assert_reaches_phase(Phase.Running, timeout=1000) + + +@mark.e2e_search_external_tls +def test_create_search_resource(mdbs: MongoDBSearch, mdbc: MongoDBCommunity): + mdbs.update() + mdbs.assert_reaches_phase(Phase.Running, timeout=300) + + +@mark.e2e_search_external_tls +def test_wait_for_community_resource_ready(mdbc: MongoDBCommunity): + mdbc.assert_reaches_phase(Phase.Running, timeout=1800) + + +@mark.e2e_search_external_tls +def test_validate_tls_connections(mdbc: MongoDBCommunity, mdbs: MongoDBSearch, namespace: str, issuer_ca_filepath: str): + with pymongo.MongoClient( + f"mongodb://{mdbc.name}-0.{mdbc.name}-svc.{namespace}.svc.cluster.local:27017/?replicaSet={mdbc.name}", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=30000, + connectTimeoutMS=20000, + ) as mongodb_client: + mongodb_info = mongodb_client.admin.command("hello") + assert mongodb_info.get("ok") == 1, "MongoDBCommunity TLS connection failed" + + with pymongo.MongoClient( + f"mongodb://{mdbs.name}-search-svc.{namespace}.svc.cluster.local:27027", + tls=True, + tlsCAFile=issuer_ca_filepath, + tlsAllowInvalidHostnames=False, + serverSelectionTimeoutMS=10000, + connectTimeoutMS=10000, + ) as search_client: + search_info = search_client.admin.command("hello") + assert search_info.get("ok") == 1, "MongoDBSearch connection failed" + + +@fixture(scope="function") +def sample_movies_helper(mdbc: MongoDBCommunity, issuer_ca_filepath: str) -> SampleMoviesSearchHelper: + return movies_search_helper.SampleMoviesSearchHelper( + SearchTester( + get_connection_string(mdbc, USER_NAME, USER_PASSWORD), + use_ssl=True, + ca_path=issuer_ca_filepath, + ) + ) + + +@mark.e2e_search_external_tls +def test_search_restore_sample_database(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.restore_sample_database() + + +@mark.e2e_search_external_tls +def test_search_create_search_index(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.create_search_index() + + +@mark.e2e_search_external_tls +def test_search_assert_search_query(sample_movies_helper: SampleMoviesSearchHelper): + sample_movies_helper.assert_search_query(retry_timeout=60) + + +def get_connection_string(mdbc: MongoDBCommunity, user_name: str, user_password: str) -> str: + return ( + f"mongodb://{user_name}:{user_password}@{mdbc.name}-0.{mdbc.name}-svc.{mdbc.namespace}.svc.cluster.local:27017/" + f"?replicaSet={mdbc.name}" + ) From 4985f0fa1f176fbac11ce648f57c53f943a79fe2 Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Tue, 19 Aug 2025 09:39:33 +0200 Subject: [PATCH 09/11] fix lint --- .../tests/search/search_community_external_mongod_tls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py index edb8a83dd..a6289f17e 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py @@ -113,8 +113,9 @@ def test_install_secrets(namespace: str, mdbs: MongoDBSearch): @mark.e2e_search_external_tls -def test_install_tls_secrets_and_configmaps(namespace: str, mdbc: MongoDBCommunity, mdbs: MongoDBSearch, issuer: str, - issuer_ca_filepath: str): +def test_install_tls_secrets_and_configmaps( + namespace: str, mdbc: MongoDBCommunity, mdbs: MongoDBSearch, issuer: str, issuer_ca_filepath: str +): create_tls_certs(issuer, namespace, mdbc.name, mdbc["spec"]["members"], secret_name=TLS_SECRET_NAME) search_service_name = f"{mdbs.name}-search-svc" From 4073818c5e6283b306b5021a8eed20239d3317aa Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Wed, 20 Aug 2025 00:47:55 +0200 Subject: [PATCH 10/11] address pr comments --- api/v1/search/mongodbsearch_types.go | 2 +- api/v1/search/zz_generated.deepcopy.go | 6 +++--- .../search_controller/search_construction.go | 15 +++++++-------- .../search_community_external_mongod_basic.py | 5 ++--- .../search_community_external_mongod_tls.py | 14 ++++++++++++-- .../controllers/replica_set_controller.go | 2 +- 6 files changed, 26 insertions(+), 18 deletions(-) diff --git a/api/v1/search/mongodbsearch_types.go b/api/v1/search/mongodbsearch_types.go index b88ce5811..791c5b1dd 100644 --- a/api/v1/search/mongodbsearch_types.go +++ b/api/v1/search/mongodbsearch_types.go @@ -69,7 +69,7 @@ type ExternalMongoDBSource struct { type ExternalMongodTLS struct { Enabled bool `json:"enabled"` // +optional - CASecretRef *userv1.SecretKeyRef `json:"caSecretRef,omitempty"` + CA *corev1.LocalObjectReference `json:"ca,omitempty"` } type Security struct { diff --git a/api/v1/search/zz_generated.deepcopy.go b/api/v1/search/zz_generated.deepcopy.go index eb015e9d1..c66322146 100644 --- a/api/v1/search/zz_generated.deepcopy.go +++ b/api/v1/search/zz_generated.deepcopy.go @@ -61,9 +61,9 @@ func (in *ExternalMongoDBSource) DeepCopy() *ExternalMongoDBSource { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ExternalMongodTLS) DeepCopyInto(out *ExternalMongodTLS) { *out = *in - if in.CASecretRef != nil { - in, out := &in.CASecretRef, &out.CASecretRef - *out = new(user.SecretKeyRef) + if in.CA != nil { + in, out := &in.CA, &out.CA + *out = new(v1.LocalObjectReference) **out = **in } } diff --git a/controllers/search_controller/search_construction.go b/controllers/search_controller/search_construction.go index fe6bbbc64..cf622c0d2 100644 --- a/controllers/search_controller/search_construction.go +++ b/controllers/search_controller/search_construction.go @@ -69,16 +69,17 @@ func (r *externalSearchResource) KeyfileSecretName() string { } func (r *externalSearchResource) IsSecurityTLSConfigEnabled() bool { - if r.spec.TLS != nil { - return r.spec.TLS.Enabled - } - return false + return r.spec.TLS != nil && r.spec.TLS.Enabled } func (r *externalSearchResource) TLSOperatorCASecretNamespacedName() types.NamespacedName { - if r.spec.TLS != nil { - return types.NamespacedName{Name: r.spec.TLS.CASecretRef.Name, Namespace: r.namespace} + if r.spec.TLS != nil && r.spec.TLS.CA != nil { + return types.NamespacedName{ + Name: r.spec.TLS.CA.Name, + Namespace: r.namespace, + } } + return types.NamespacedName{} } @@ -135,8 +136,6 @@ func (r *mdbcSearchResource) DatabaseServiceName() string { return r.db.ServiceName() } -// replace with a validate method that is always true for external mongodb - func (r *mdbcSearchResource) IsSecurityTLSConfigEnabled() bool { return r.db.Spec.Security.TLS.Enabled } diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py index f2298c069..3b6e326b9 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_basic.py @@ -32,8 +32,8 @@ def mdbc(namespace: str) -> MongoDBCommunity: namespace=namespace, ) - # if try_load(resource): - # return resource + if try_load(resource): + return resource mongot_host = f"{MDBC_RESOURCE_NAME}-search-svc.{namespace}.svc.cluster.local:27027" if "additionalMongodConfig" not in resource["spec"]: @@ -41,7 +41,6 @@ def mdbc(namespace: str) -> MongoDBCommunity: if "setParameter" not in resource["spec"]["additionalMongodConfig"]: resource["spec"]["additionalMongodConfig"]["setParameter"] = {} - # Update the setParameter section resource["spec"]["additionalMongodConfig"]["setParameter"].update( { "mongotHost": mongot_host, diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py index a6289f17e..0f3b10294 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py @@ -1,5 +1,5 @@ import pymongo -from kubetester import create_or_update_secret, try_load +from kubetester import create_or_update_secret, create_object_from_dict, try_load from kubetester.certs import create_tls_certs from kubetester.kubetester import fixture as yaml_fixture from kubetester.mongodb_community import MongoDBCommunity @@ -25,6 +25,7 @@ MDBC_RESOURCE_NAME = "mdbc-rs" TLS_SECRET_NAME = "tls-secret" +TLS_CA_CONFIGMAP_NAME = "tls-ca-configmap" TLS_CA_SECRET_NAME = "tls-ca-secret" MDBS_TLS_SECRET_NAME = "mdbs-tls-secret" @@ -83,7 +84,7 @@ def mdbs(namespace: str, mdbc: MongoDBCommunity) -> MongoDBSearch: "keyFileSecretRef": {"name": f"{mdbc.name}-keyfile"}, "tls": { "enabled": True, - "caSecretRef": {"name": TLS_CA_SECRET_NAME, "key": "ca.crt"}, + "ca": {"name": TLS_CA_CONFIGMAP_NAME}, }, } } @@ -130,8 +131,17 @@ def test_install_tls_secrets_and_configmaps( ) ca = open(issuer_ca_filepath).read() + create_or_update_secret(namespace=namespace, name=TLS_CA_SECRET_NAME, data={"ca.crt": ca}) + ca_configmap = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": {"name": TLS_CA_CONFIGMAP_NAME, "namespace": namespace}, + "data": {"ca.crt": ca}, + } + create_object_from_dict(ca_configmap, namespace) + @mark.e2e_search_external_tls def test_create_database_resource(mdbc: MongoDBCommunity): diff --git a/mongodb-community-operator/controllers/replica_set_controller.go b/mongodb-community-operator/controllers/replica_set_controller.go index 52bbcb938..db3690a02 100644 --- a/mongodb-community-operator/controllers/replica_set_controller.go +++ b/mongodb-community-operator/controllers/replica_set_controller.go @@ -90,7 +90,7 @@ func NewReconciler(mgr manager.Manager, mongodbRepoUrl, mongodbImage, mongodbIma func findMdbcForSearch(ctx context.Context, rawObj k8sClient.Object) []reconcile.Request { mdbSearch := rawObj.(*searchv1.MongoDBSearch) if mdbSearch.GetMongoDBResourceRef() == nil { - return nil // maybe empty array + return nil } return []reconcile.Request{ {NamespacedName: types.NamespacedName{Namespace: mdbSearch.GetMongoDBResourceRef().Namespace, Name: mdbSearch.GetMongoDBResourceRef().Name}}, From 9059f18ad532609402361f7201d011c7b8836864 Mon Sep 17 00:00:00 2001 From: Anand Singh Date: Wed, 20 Aug 2025 12:22:05 +0200 Subject: [PATCH 11/11] update test --- .../search/search_community_external_mongod_tls.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py index 0f3b10294..8542f242b 100644 --- a/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py +++ b/docker/mongodb-kubernetes-tests/tests/search/search_community_external_mongod_tls.py @@ -1,5 +1,5 @@ import pymongo -from kubetester import create_or_update_secret, create_object_from_dict, try_load +from kubetester import create_or_update_configmap, create_or_update_secret, try_load from kubetester.certs import create_tls_certs from kubetester.kubetester import fixture as yaml_fixture from kubetester.mongodb_community import MongoDBCommunity @@ -134,13 +134,7 @@ def test_install_tls_secrets_and_configmaps( create_or_update_secret(namespace=namespace, name=TLS_CA_SECRET_NAME, data={"ca.crt": ca}) - ca_configmap = { - "apiVersion": "v1", - "kind": "ConfigMap", - "metadata": {"name": TLS_CA_CONFIGMAP_NAME, "namespace": namespace}, - "data": {"ca.crt": ca}, - } - create_object_from_dict(ca_configmap, namespace) + create_or_update_configmap(namespace, TLS_CA_CONFIGMAP_NAME, {"ca.crt": ca}) @mark.e2e_search_external_tls