Add sriov dp admission controller to dpu operator

Makefile

@@ -218,11 +218,12 @@ REGISTRY ?= $(shell hostname)
 # development.
 DPU_OPERATOR_IMAGE := $(REGISTRY):5000/dpu-operator:dev
 DPU_DAEMON_IMAGE := $(REGISTRY):5000/dpu-daemon:dev
+ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT ?= $(shell kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 --w=0)
 MARVELL_VSP_IMAGE := $(REGISTRY):5000/mrvl-vsp:dev
 
 .PHONY: local-deploy-prep
 prep-local-deploy: tools
-	./bin/config -registry-url $(REGISTRY) -template-file config/dev/local-images-template.yaml -output-file bin/local-images.yaml
+	./bin/config -registry-url $(REGISTRY) -admissions-controllers-ca-crt $(ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT) -template-file config/dev/local-images-template.yaml -output-file bin/local-images.yaml
 	cp config/dev/kustomization.yaml bin
 
 .PHONY: local-deploy
@@ -245,13 +246,13 @@ local-buildx: ## Build all container images necessary to run the whole operator
 	mkdir -p $(GO_CONTAINER_CACHE)
 	buildah manifest rm $(DPU_OPERATOR_IMAGE)-manifest || true
 	buildah manifest create $(DPU_OPERATOR_IMAGE)-manifest
-	buildah build --authfile /root/config.json --manifest $(DPU_OPERATOR_IMAGE)-manifest --platform linux/amd64,linux/arm64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.rhel -t $(DPU_OPERATOR_IMAGE)
+	buildah build --authfile /root/config.json --manifest $(DPU_OPERATOR_IMAGE)-manifest --platform linux/amd64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.rhel -t $(DPU_OPERATOR_IMAGE)
 	buildah manifest rm $(DPU_DAEMON_IMAGE)-manifest || true
 	buildah manifest create $(DPU_DAEMON_IMAGE)-manifest
-	buildah build --authfile /root/config.json --manifest $(DPU_DAEMON_IMAGE)-manifest --platform linux/amd64,linux/arm64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.daemon.rhel -t $(DPU_DAEMON_IMAGE)
+	buildah build --authfile /root/config.json --manifest $(DPU_DAEMON_IMAGE)-manifest --platform linux/amd64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.daemon.rhel -t $(DPU_DAEMON_IMAGE)
 	buildah manifest rm $(MARVELL_VSP_IMAGE)-manifest || true
 	buildah manifest create $(MARVELL_VSP_IMAGE)-manifest
-	buildah build --authfile /root/config.json --manifest $(MARVELL_VSP_IMAGE)-manifest --platform linux/amd64,linux/arm64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.mrvlVSP.rhel -t $(MARVELL_VSP_IMAGE)
+	buildah build --authfile /root/config.json --manifest $(MARVELL_VSP_IMAGE)-manifest --platform linux/amd64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.mrvlVSP.rhel -t $(MARVELL_VSP_IMAGE)
 
 .PHONY: local-pushx
 local-pushx: ## Push all container images necessary to run the whole operator

README.md

@@ -1,4 +1,4 @@
-# DPU Operator
+ # DPU Operator
 
 This operator will manage and configure data processing unit (DPUs) to be used in accelerating/offloading k8s networking functions.
 

bundle/manifests/dpu-operator.clusterserviceversion.yaml

@@ -39,6 +39,7 @@ metadata:
       ]
     capabilities: Basic Install
     createdAt: "2024-10-15T19:06:53Z"
+    createdAt: "2024-10-11T15:10:43Z"
     features.operators.openshift.io/cnf: "false"
     features.operators.openshift.io/cni: "true"
     features.operators.openshift.io/csi: "false"
@@ -121,6 +122,18 @@ spec:
           - patch
           - update
           - watch
+        - apiGroups:
+          - apps
+          resources:
+          - deployments
+          verbs:
+          - create
+          - delete
+          - get
+          - list
+          - patch
+          - update
+          - watch
         - apiGroups:
           - config.openshift.io
           resources:

cmd/main.go

@@ -103,8 +103,13 @@ func main() {
 		setupLog.Error(err, "Failed to set DPU_DAEMON_IMAGE env var")
 		os.Exit(1)
 	}
+	injectorWebhookCA := os.Getenv("ADMISSION_CONTROLLERS_CA_CRT")
+	if injectorWebhookCA == "" {
+		setupLog.Error(err, "Failed to set ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT env var")
+		os.Exit(1)
+	}
 
-	b := controller.NewDpuOperatorConfigReconciler(mgr.GetClient(), mgr.GetScheme(), dpuDaemonImage, vspImages)
+	b := controller.NewDpuOperatorConfigReconciler(mgr.GetClient(), mgr.GetScheme(), dpuDaemonImage, vspImages, injectorWebhookCA)
 
 	if value, ok := os.LookupEnv("IMAGE_PULL_POLICIES"); ok {
 		b = b.WithImagePullPolicy(value)

config/dev/local-images-template.yaml

@@ -24,6 +24,8 @@ spec:
           value: {{ .RegistryURL }}:5000/intel_vsp:dev
         - name: MarvellVspImage
           value: {{ .RegistryURL }}:5000/mrvl-vsp:dev
+        - name: ADMISSION_CONTROLLERS_CA_CRT
+          value: {{ .AdmissionControllersCaCrt }}
         - name: IMAGE_PULL_POLICIES
           value: Always
         image: {{ .RegistryURL }}:5000/dpu-operator:dev

config/rbac/auth_proxy_client_clusterrole.yaml

@@ -14,3 +14,4 @@ rules:
   - "/metrics"
   verbs:
   - get
+

config/rbac/role.yaml

@@ -16,6 +16,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - '*'
 - apiGroups:
   - ""
   resources:
@@ -28,6 +34,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - '*'
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  verbs:
+  - '*'
 - apiGroups:
   - apiextensions.k8s.io
   resources:
@@ -46,6 +64,18 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - create
+  - delete
+  - get
+  - list
+  - patch
+  - update
+  - watch
 - apiGroups:
   - config.openshift.io
   resources:

hack/make_fast.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# Define a function to check OpenShift cluster nodes
+check_ocp() {
+    local KUBECONFIG_PATH=$1
+
+    export KUBECONFIG=$KUBECONFIG_PATH
+
+    if oc get nodes; then
+        return 0  
+    else
+        return 1  
+    fi
+}
+
+deploy_cluster_and_dpu_operator() {
+   bash hack/prepare.sh
+   bash hack/ipu_host_deploy.sh
+   bash hack/ipu_deploy.sh
+   bash hack/deploy_traffic_flow_tests.sh
+}
+
+deploy_local_dpu_operator() {
+   echo "deploying local"
+   bash hack/prepare.sh
+   #bash hack/ipu_host_deploy_post.sh
+   #bash hack/ipu_deploy_post.sh
+   #bash hack/deploy_traffic_flow_tests.sh
+
+}
+
+check_ocp "/root/kubeconfig.ocpcluster"
+result_ocp=$?  # Store the return value (0 for true, 1 for false)
+
+check_ocp "/root/kubeconfig.microshift"
+result_microshift=$?  # Store the return value (0 for true, 1 for false)
+
+# Check if both OCP and MicroShift checks succeeded
+if [ $result_ocp -eq 0 ] && [ $result_microshift -eq 0 ]; then
+    echo "Both OCP and MicroShift cluster checks succeeded!"
+    deploy_local_dpu_operator
+else
+    echo "Cluster config is not proper... deploying cluster and dpu operator"
+    #deploy_cluster_and_dpu_operator
+fi
+

internal/controller/bindata/webhook/000_service.yaml

@@ -0,0 +1,13 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: network-resources-injector-service
+  namespace: openshift-dpu-operator
+  annotations:
+    service.alpha.openshift.io/serving-cert-secret-name: network-resources-injector-secret
+spec:
+  ports:
+  - port: 443
+    targetPort: 8443
+  selector:
+    app: network-resources-injector

internal/controller/bindata/webhook/00_serviceaccount.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  namespace: openshift-dpu-operator
+  name: network-resources-injector-sa
+

internal/controller/bindata/webhook/01_secret.yaml

@@ -0,0 +1,8 @@
+apiVersion: v1
+kind: Secret
+metadata:
+  name: network-resources-injector-sa-secret
+  namespace: openshift-dpu-operator
+  annotations:
+    kubernetes.io/service-account.name: network-resources-injector-sa
+type: kubernetes.io/service-account-token

internal/controller/bindata/webhook/02-clusterrole-network-resources-injector.yaml

@@ -0,0 +1,48 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: network-resources-injector
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - serviceaccounts
+  - pods
+  verbs:
+  - get
+  - list
+  - watch
+  - create
+  - delete
+- apiGroups:
+  - k8s.cni.cncf.io
+  resources:
+  - network-attachment-definitions
+  verbs:
+  - 'watch'
+  - 'list'
+  - 'get'
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - 'get'
+- apiGroups:
+  - apps
+  resources:
+  - deployments
+  verbs:
+  - 'watch'
+  - 'list'
+  - 'get'
+- apiGroups:
+  - security.openshift.io
+  resourceNames:
+  - anyuid
+  - hostnetwork
+  - privileged
+  resources:
+  - securitycontextconstraints
+  verbs:
+  - 'use'

internal/controller/bindata/webhook/03_clusterrole_secrets.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: network-resources-injector-secrets 
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - secrets
+  verbs:
+  - '*'

internal/controller/bindata/webhook/04-clusterrole-webhook-configs.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: network-resources-injector-webhook-configs
+rules:
+- apiGroups:
+  - admissionregistration.k8s.io
+  resources:
+  - mutatingwebhookconfigurations
+  verbs:
+  - '*'

internal/controller/bindata/webhook/05-clusterrole-service.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: network-resources-injector-service
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - services
+  verbs:
+  - '*'

internal/controller/bindata/webhook/06-clusterrole-configmaps.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: network-resources-injector-configmaps
+rules:
+- apiGroups:
+  - ""
+  resources:
+  - configmaps
+  verbs:
+  - 'get'

internal/controller/bindata/webhook/07-clusterrolebinding-network-resources-injector-role-binding.yaml

@@ -0,0 +1,11 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: network-resources-injector-role-binding
+roleRef:
+  kind: ClusterRole
+  name: network-resources-injector
+subjects:
+- kind: ServiceAccount
+  name: network-resources-injector-sa
+  namespace: openshift-dpu-operator

internal/controller/bindata/webhook/08-clusterrolebinding-secrets.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: network-resources-injector-secrets-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: network-resources-injector-secrets
+subjects:
+- kind: ServiceAccount
+  name: network-resources-injector-sa
+  namespace: openshift-dpu-operator

internal/controller/bindata/webhook/09-clusterrolebinding-webhook-configs.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: network-resources-injector-webhook-configs-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: network-resources-injector-webhook-configs
+subjects:
+- kind: ServiceAccount
+  name: network-resources-injector-sa
+  namespace: openshift-dpu-operator

internal/controller/bindata/webhook/10-clusterrolebinding-service.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: network-resources-injector-service-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: network-resources-injector-service
+subjects:
+- kind: ServiceAccount
+  name: network-resources-injector-sa
+  namespace: openshift-dpu-operator

internal/controller/bindata/webhook/11-clusterrolebinding-configmaps.yaml

@@ -0,0 +1,12 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: network-resources-injector-configmaps-role-binding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: network-resources-injector-configmaps
+subjects:
+- kind: ServiceAccount
+  name: network-resources-injector-sa
+  namespace: openshift-dpu-operator