diff --git a/go.mod b/go.mod index 2dd1bcea..4226f45d 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,9 @@ require ( k8s.io/client-go v0.26.2 k8s.io/klog/v2 v2.90.1 k8s.io/metrics v0.26.2 + k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 sigs.k8s.io/custom-metrics-apiserver v0.0.0 + sigs.k8s.io/structured-merge-diff/v4 v4.2.3 ) replace sigs.k8s.io/custom-metrics-apiserver => sigs.k8s.io/custom-metrics-apiserver v1.25.1-0.20230306170449-63d8c93851f3 @@ -107,9 +109,7 @@ require ( k8s.io/component-base v0.26.2 // indirect k8s.io/kms v0.26.2 // indirect k8s.io/kube-openapi v0.0.0-20230303024457-afdc3dddf62d // indirect - k8s.io/utils v0.0.0-20230220204549-a5ecb0141aa5 // indirect sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.35 // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect - sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/pkg/controller/queuejobresources/genericresource/genericresource.go b/pkg/controller/queuejobresources/genericresource/genericresource.go index a1057f76..5cdad5fc 100644 --- a/pkg/controller/queuejobresources/genericresource/genericresource.go +++ b/pkg/controller/queuejobresources/genericresource/genericresource.go @@ -43,7 +43,8 @@ import ( "k8s.io/client-go/restmapper" ) -var appwrapperJobName = "appwrapper.mcad.ibm.com" +var appwrapperJobLabelName = "appwrapper.mcad.ibm.com" +var appwrapperJobLabelNamespace = "appwrapper.mcad.ibm.com/namespace" var resourceName = "resourceName" var appWrapperKind = arbv1.SchemeGroupVersion.WithKind("AppWrapper") @@ -166,7 +167,7 @@ func (gr *GenericResources) Cleanup(aw *arbv1.AppWrapper, awr *arbv1.AppWrapperG } // Get the resource to see if it exists in the AppWrapper namespace - labelSelector := fmt.Sprintf("%s=%s, %s=%s", appwrapperJobName, aw.Name, resourceName, unstruct.GetName()) + labelSelector := fmt.Sprintf("%s=%s, %s=%s", appwrapperJobLabelName, aw.Name, appwrapperJobLabelNamespace, aw.Namespace) inEtcd, err := dclient.Resource(rsrc).Namespace(aw.Namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector}) if err != nil { return name, gvk, err @@ -187,7 +188,7 @@ func (gr *GenericResources) Cleanup(aw *arbv1.AppWrapper, awr *arbv1.AppWrapperG return name, gvk, err } } else { - klog.Warningf("[Cleanup] %s/%s not found using label selector: %s.\n", name, namespace, labelSelector) + klog.Warningf("[Cleanup] %s/%s not found using label selector: %s.\n", namespace, name, labelSelector) } return name, gvk, err @@ -297,18 +298,19 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra } else { labels = unstruct.GetLabels() } - labels[appwrapperJobName] = aw.Name + labels[appwrapperJobLabelName] = aw.Name + labels[appwrapperJobLabelNamespace] = aw.Namespace labels[resourceName] = unstruct.GetName() unstruct.SetLabels(labels) // Add labels to pod template if one exists. podTemplateFound := addLabelsToPodTemplateField(&unstruct, labels) if !podTemplateFound { - klog.V(4).Infof("[SyncQueueJob] No pod template spec exists for resource: %s to add labels.", name) + klog.V(4).Infof("[SyncQueueJob] No pod template spec exists for resource: %s/%s to add labels.", namespace, name) } - // Get the resource to see if it exists - labelSelector := fmt.Sprintf("%s=%s, %s=%s", appwrapperJobName, aw.Name, resourceName, unstruct.GetName()) + // Get the resource to see if it exists + labelSelector := fmt.Sprintf("%s=%s, %s=%s", appwrapperJobLabelName, aw.Name, appwrapperJobLabelNamespace, aw.Namespace) inEtcd, err := dclient.Resource(rsrc).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector}) if err != nil { return []*v1.Pod{}, err @@ -329,7 +331,7 @@ func (gr *GenericResources) SyncQueueJob(aw *arbv1.AppWrapper, awr *arbv1.AppWra if errors.IsAlreadyExists(err) { klog.V(4).Infof("%v\n", err.Error()) } else { - klog.Errorf("Error creating the object `%v`, the error is `%v`", newName, errors.ReasonForError(err)) + klog.Errorf("Error creating the object `%s/%s`, the error is `%v`", namespace, newName, errors.ReasonForError(err)) return []*v1.Pod{}, err } } @@ -499,7 +501,7 @@ func deleteObject(namespaced bool, namespace string, name string, rsrc schema.Gr } if err != nil && !errors.IsNotFound(err) { - klog.Errorf("[deleteObject] Error deleting the object `%v`, the error is `%v`.", name, errors.ReasonForError(err)) + klog.Errorf("[deleteObject] Error deleting the object `%v`, in namespace %v, the error is `%v`.", name, namespace, errors.ReasonForError(err)) return err } else { klog.V(4).Infof("[deleteObject] Resource `%v` deleted.\n", name) @@ -531,7 +533,7 @@ func GetListOfPodResourcesFromOneGenericItem(awr *arbv1.AppWrapperGenericResourc klog.V(8).Infof("[GetListOfPodResourcesFromOneGenericItem] Requested total allocation resource from 1 pod `%v`.\n", podTotalresource) } - // Addd individual pods to results + // Add individual pods to results var replicaCount int = int(replicas) for i := 0; i < replicaCount; i++ { podResourcesList = append(podResourcesList, podTotalresource) @@ -623,7 +625,7 @@ func getContainerResources(container v1.Container, replicas float64) *clustersta } // returns status of an item present in etcd -func (gr *GenericResources) IsItemCompleted(awgr *arbv1.AppWrapperGenericResource, namespace string, appwrapperName string, genericItemName string) (completed bool) { +func (gr *GenericResources) IsItemCompleted(awgr *arbv1.AppWrapperGenericResource, appwrapperNamespace string, appwrapperName string, genericItemName string) (completed bool) { dd := gr.clients.Discovery() apigroups, err := restmapper.GetAPIGroupResources(dd) if err != nil { @@ -654,8 +656,8 @@ func (gr *GenericResources) IsItemCompleted(awgr *arbv1.AppWrapperGenericResourc return false } - labelSelector := fmt.Sprintf("%s=%s", appwrapperJobName, appwrapperName) - inEtcd, err := dclient.Resource(rsrc).Namespace(namespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector}) + labelSelector := fmt.Sprintf("%s=%s, %s=%s", appwrapperJobLabelName, appwrapperName, appwrapperJobLabelNamespace, appwrapperNamespace) + inEtcd, err := dclient.Resource(rsrc).Namespace(appwrapperNamespace).List(context.Background(), metav1.ListOptions{LabelSelector: labelSelector}) if err != nil { klog.Errorf("[IsItemCompleted] Error listing object: %v", err) return false @@ -675,7 +677,7 @@ func (gr *GenericResources) IsItemCompleted(awgr *arbv1.AppWrapperGenericResourc } } if !validAwOwnerRef { - klog.Warningf("[IsItemCompleted] Item owner name %v does match appwrappper name %v in namespace %v", unstructuredObjectName, appwrapperName, namespace) + klog.Warningf("[IsItemCompleted] Item owner name %v does match appwrappper name %v in namespace %v", unstructuredObjectName, appwrapperName, appwrapperNamespace) continue } diff --git a/test/e2e-kuttl-deployment-01/steps/01-assert.yaml b/test/e2e-kuttl-deployment-01/steps/01-assert.yaml index cbf35849..1d7fc1d1 100644 --- a/test/e2e-kuttl-deployment-01/steps/01-assert.yaml +++ b/test/e2e-kuttl-deployment-01/steps/01-assert.yaml @@ -15,6 +15,7 @@ metadata: labels: app: no-quota-deployment-01 appwrapper.mcad.ibm.com: no-quota-deployment-01 + appwrapper.mcad.ibm.com/namespace: start-up resourceName: no-quota-deployment-01 status: availableReplicas: 1 diff --git a/test/e2e-kuttl-deployment-01/steps/03-assert.yaml b/test/e2e-kuttl-deployment-01/steps/03-assert.yaml index fb5d6ba6..73d11699 100644 --- a/test/e2e-kuttl-deployment-01/steps/03-assert.yaml +++ b/test/e2e-kuttl-deployment-01/steps/03-assert.yaml @@ -13,6 +13,7 @@ metadata: namespace: start-up labels: appwrapper.mcad.ibm.com: hold-completion-job-03 + appwrapper.mcad.ibm.com/namespace: start-up resourceName: hold-completion-job-03-01 status: conditions: @@ -20,20 +21,22 @@ status: type: Complete succeeded: 1 --- -apiVersion: v1 -kind: Pod -metadata: +apiVersion: v1 +kind: Pod +metadata: namespace: start-up - labels: - appwrapper.mcad.ibm.com: hold-completion-job-03 - job-name: hold-completion-job-03-01 - resourceName: hold-completion-job-03-01 + labels: + appwrapper.mcad.ibm.com: hold-completion-job-03 + appwrapper.mcad.ibm.com/namespace: start-up + job-name: hold-completion-job-03-01 + resourceName: hold-completion-job-03-01 --- -apiVersion: v1 -kind: Pod -metadata: +apiVersion: v1 +kind: Pod +metadata: namespace: start-up - labels: - appwrapper.mcad.ibm.com: hold-completion-job-03 - job-name: hold-completion-job-03-02 - resourceName: hold-completion-job-03-02 + labels: + appwrapper.mcad.ibm.com: hold-completion-job-03 + appwrapper.mcad.ibm.com/namespace: start-up + job-name: hold-completion-job-03-02 + resourceName: hold-completion-job-03-02 diff --git a/test/e2e-kuttl-deployment-01/steps/07-assert.yaml b/test/e2e-kuttl-deployment-01/steps/07-assert.yaml index dc0ade42..518320cb 100644 --- a/test/e2e-kuttl-deployment-01/steps/07-assert.yaml +++ b/test/e2e-kuttl-deployment-01/steps/07-assert.yaml @@ -7,12 +7,12 @@ metadata: status: state: Running --- -apiVersion: v1 -kind: Pod -metadata: +apiVersion: v1 +kind: Pod +metadata: namespace: start-up - labels: - appwrapper.mcad.ibm.com: no-quota-job-06 - job-name: no-quota-job-06 + labels: + appwrapper.mcad.ibm.com: no-quota-job-06 + appwrapper.mcad.ibm.com/namespace: start-up + job-name: no-quota-job-06 resourceName: no-quota-job-06 - diff --git a/test/e2e-kuttl-deployment-02/steps/02-assert.yaml b/test/e2e-kuttl-deployment-02/steps/02-assert.yaml index aa9a6f7e..297d1c67 100644 --- a/test/e2e-kuttl-deployment-02/steps/02-assert.yaml +++ b/test/e2e-kuttl-deployment-02/steps/02-assert.yaml @@ -15,6 +15,7 @@ metadata: labels: app: no-quota-deployment-02 appwrapper.mcad.ibm.com: no-quota-deployment-02 + appwrapper.mcad.ibm.com/namespace: start-up-02 resourceName: no-quota-deployment-02 status: availableReplicas: 1 diff --git a/test/e2e-kuttl-deployment-03/steps/02-assert.yaml b/test/e2e-kuttl-deployment-03/steps/02-assert.yaml index 8551bdf6..758038ed 100644 --- a/test/e2e-kuttl-deployment-03/steps/02-assert.yaml +++ b/test/e2e-kuttl-deployment-03/steps/02-assert.yaml @@ -15,6 +15,7 @@ metadata: labels: app: no-quota-deployment-02 appwrapper.mcad.ibm.com: no-quota-deployment-02 + appwrapper.mcad.ibm.com/namespace: start-up-03 resourceName: no-quota-deployment-02 status: availableReplicas: 1 diff --git a/test/e2e-kuttl/quota-errors/03-assert.yaml b/test/e2e-kuttl/quota-errors/03-assert.yaml index 629127c4..26c28c55 100644 --- a/test/e2e-kuttl/quota-errors/03-assert.yaml +++ b/test/e2e-kuttl/quota-errors/03-assert.yaml @@ -18,7 +18,8 @@ metadata: labels: app: deployment-silver-lo-pri-1replica appwrapper.mcad.ibm.com: deployment-silver-lo-pri-1replica - resourceName: deployment-silver-lo-pri-1replica + appwrapper.mcad.ibm.com/namespace: quota-errors + resourceName: deployment-silver-lo-pri-1replica status: availableReplicas: 1 observedGeneration: 1 diff --git a/test/e2e/queue.go b/test/e2e/queue.go index 2fd6f1b3..41341e34 100644 --- a/test/e2e/queue.go +++ b/test/e2e/queue.go @@ -200,7 +200,7 @@ var _ = Describe("AppWrapper E2E Test", func() { appwrappersPtr := &appwrappers defer cleanupTestObjectsPtr(context, appwrappersPtr) - aw := createDeploymentAW(context, "aw-deployment-3") + aw := createDeploymentAW(context, "aw-deployment-3", "test") appwrappers = append(appwrappers, aw) fmt.Fprintf(GinkgoWriter, "[e2e] Awaiting %d pods running for AW %s.\n", aw.Spec.SchedSpec.MinAvailable, aw.Name) @@ -208,6 +208,24 @@ var _ = Describe("AppWrapper E2E Test", func() { Expect(err).NotTo(HaveOccurred()) }) + It("Create Two AppWrappers Same Name Different Namespaces - Deployment Only - 3 Pods Each", func() { + fmt.Fprintf(os.Stdout, "[e2e] Create Two AppWrappers Same Name Different Namespaces - Deployment Only 3 Pods Each - Started.\n") + context := initTestContext() + var appwrappers []*arbv1.AppWrapper + appwrappersPtr := &appwrappers + defer cleanupTestObjectsPtr(context, appwrappersPtr) + + namespaces := []string{"nstest1", "nstest2"} + for _, ns := range namespaces { + aw := createDeploymentAW(context, "aw-deployment-3", ns) + appwrappers = append(appwrappers, aw) + + fmt.Fprintf(GinkgoWriter, "[e2e] Awaiting %d pods running for AW %s in namespace %s.\n", aw.Spec.SchedSpec.MinAvailable, aw.Name, ns) + err := waitAWPodsReady(context, aw) + Expect(err).NotTo(HaveOccurred()) + } + }) + It("Create AppWrapper - Generic Deployment Only - 3 pods", func() { fmt.Fprintf(os.Stdout, "[e2e] Create AppWrapper - Generic Deployment Only - 3 pods - Started.\n") context := initTestContext() diff --git a/test/e2e/util.go b/test/e2e/util.go index 363a208e..aa91fd6e 100644 --- a/test/e2e/util.go +++ b/test/e2e/util.go @@ -646,12 +646,12 @@ func createJobAWWithInitContainer(context *context, name string, requeuingTimeIn return appwrapper } -func createDeploymentAW(context *context, name string) *arbv1.AppWrapper { - rb := []byte(`{"apiVersion": "apps/v1", +func createDeploymentAW(context *context, name string, namespace string) *arbv1.AppWrapper { + rb := []byte(fmt.Sprintf(`{"apiVersion": "apps/v1", "kind": "Deployment", "metadata": { "name": "aw-deployment-3", - "namespace": "test", + "namespace": "%s", "labels": { "app": "aw-deployment-3" } @@ -686,13 +686,13 @@ func createDeploymentAW(context *context, name string) *arbv1.AppWrapper { ] } } - }} `) + }} `, namespace)) var schedSpecMin int = 3 aw := &arbv1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ Name: name, - Namespace: context.namespace, + Namespace: namespace, }, Spec: arbv1.AppWrapperSpec{ SchedSpec: arbv1.SchedulingSpecTemplate{ @@ -711,7 +711,7 @@ func createDeploymentAW(context *context, name string) *arbv1.AppWrapper { }, } - appwrapper, err := context.karclient.WorkloadV1beta1().AppWrappers(context.namespace).Create(context.ctx, aw, metav1.CreateOptions{}) + appwrapper, err := context.karclient.WorkloadV1beta1().AppWrappers(namespace).Create(context.ctx, aw, metav1.CreateOptions{}) Expect(err).NotTo(HaveOccurred()) return appwrapper @@ -1512,6 +1512,7 @@ func createGenericServiceAWWithNoStatus(context *context, name string) *arbv1.Ap "metadata": { "labels": { "appwrapper.mcad.ibm.com": "test-dep-job-item", + "appwrapper.mcad.ibm.com/namespace": "test", "resourceName": "test-dep-job-item-svc" }, "name": "test-dep-job-item-svc",