Skip to content

Commit

Permalink
Add sriov dp admission controller to dpu operator
Browse files Browse the repository at this point in the history
  • Loading branch information
vrindle committed Dec 9, 2024
1 parent 99f3e30 commit 9e90fda
Show file tree
Hide file tree
Showing 16 changed files with 313 additions and 19 deletions.
9 changes: 5 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -218,11 +218,12 @@ REGISTRY ?= $(shell hostname)
# development.
DPU_OPERATOR_IMAGE := $(REGISTRY):5000/dpu-operator:dev
DPU_DAEMON_IMAGE := $(REGISTRY):5000/dpu-daemon:dev
ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT ?= $(shell kubectl get configmap -n kube-system extension-apiserver-authentication -o=jsonpath='{.data.client-ca-file}' | base64 --w=0)
MARVELL_VSP_IMAGE := $(REGISTRY):5000/mrvl-vsp:dev

.PHONY: local-deploy-prep
prep-local-deploy: tools
./bin/config -registry-url $(REGISTRY) -template-file config/dev/local-images-template.yaml -output-file bin/local-images.yaml
./bin/config -registry-url $(REGISTRY) -admissions-controllers-ca-crt $(ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT) -template-file config/dev/local-images-template.yaml -output-file bin/local-images.yaml
cp config/dev/kustomization.yaml bin

.PHONY: local-deploy
Expand All @@ -245,13 +246,13 @@ local-buildx: ## Build all container images necessary to run the whole operator
mkdir -p $(GO_CONTAINER_CACHE)
buildah manifest rm $(DPU_OPERATOR_IMAGE)-manifest || true
buildah manifest create $(DPU_OPERATOR_IMAGE)-manifest
buildah build --authfile /root/config.json --manifest $(DPU_OPERATOR_IMAGE)-manifest --platform linux/amd64,linux/arm64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.rhel -t $(DPU_OPERATOR_IMAGE)
buildah build --authfile /root/config.json --manifest $(DPU_OPERATOR_IMAGE)-manifest --platform linux/amd64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.rhel -t $(DPU_OPERATOR_IMAGE)
buildah manifest rm $(DPU_DAEMON_IMAGE)-manifest || true
buildah manifest create $(DPU_DAEMON_IMAGE)-manifest
buildah build --authfile /root/config.json --manifest $(DPU_DAEMON_IMAGE)-manifest --platform linux/amd64,linux/arm64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.daemon.rhel -t $(DPU_DAEMON_IMAGE)
buildah build --authfile /root/config.json --manifest $(DPU_DAEMON_IMAGE)-manifest --platform linux/amd64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.daemon.rhel -t $(DPU_DAEMON_IMAGE)
buildah manifest rm $(MARVELL_VSP_IMAGE)-manifest || true
buildah manifest create $(MARVELL_VSP_IMAGE)-manifest
buildah build --authfile /root/config.json --manifest $(MARVELL_VSP_IMAGE)-manifest --platform linux/amd64,linux/arm64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.mrvlVSP.rhel -t $(MARVELL_VSP_IMAGE)
buildah build --authfile /root/config.json --manifest $(MARVELL_VSP_IMAGE)-manifest --platform linux/amd64 -v $(GO_CONTAINER_CACHE):/go:z -f Dockerfile.mrvlVSP.rhel -t $(MARVELL_VSP_IMAGE)

.PHONY: local-pushx
local-pushx: ## Push all container images necessary to run the whole operator
Expand Down
13 changes: 13 additions & 0 deletions bundle/manifests/dpu-operator.clusterserviceversion.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ metadata:
]
capabilities: Basic Install
createdAt: "2024-10-15T19:06:53Z"
createdAt: "2024-10-11T15:10:43Z"
features.operators.openshift.io/cnf: "false"
features.operators.openshift.io/cni: "true"
features.operators.openshift.io/csi: "false"
Expand Down Expand Up @@ -121,6 +122,18 @@ spec:
- patch
- update
- watch
- apiGroups:
- apps
resources:
- deployments
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- config.openshift.io
resources:
Expand Down
7 changes: 6 additions & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -103,8 +103,13 @@ func main() {
setupLog.Error(err, "Failed to set DPU_DAEMON_IMAGE env var")
os.Exit(1)
}
injectorWebhookCA := os.Getenv("ADMISSION_CONTROLLERS_CA_CRT")
if injectorWebhookCA == "" {
setupLog.Error(err, "Failed to set ADMISSION_CONTROLLERS_CERTIFICATES_INJECTOR_CA_CRT env var")
os.Exit(1)
}

b := controller.NewDpuOperatorConfigReconciler(mgr.GetClient(), mgr.GetScheme(), dpuDaemonImage, vspImages)
b := controller.NewDpuOperatorConfigReconciler(mgr.GetClient(), mgr.GetScheme(), dpuDaemonImage, vspImages, injectorWebhookCA)

if value, ok := os.LookupEnv("IMAGE_PULL_POLICIES"); ok {
b = b.WithImagePullPolicy(value)
Expand Down
2 changes: 2 additions & 0 deletions config/dev/local-images-template.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ spec:
value: {{ .RegistryURL }}:5000/intel_vsp:dev
- name: MarvellVspImage
value: {{ .RegistryURL }}:5000/mrvl-vsp:dev
- name: ADMISSION_CONTROLLERS_CA_CRT
value: {{ .AdmissionControllersCaCrt }}
- name: IMAGE_PULL_POLICIES
value: Always
image: {{ .RegistryURL }}:5000/dpu-operator:dev
Expand Down
1 change: 1 addition & 0 deletions config/rbac/auth_proxy_client_clusterrole.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,3 +14,4 @@ rules:
- "/metrics"
verbs:
- get

24 changes: 24 additions & 0 deletions config/rbac/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,18 @@ rules:
- patch
- update
- watch
- apiGroups:
- ""
resources:
- mutatingwebhookconfigurations
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- ""
resources:
Expand Down Expand Up @@ -46,6 +58,18 @@ rules:
- patch
- update
- watch
- apiGroups:
- apps
resources:
- deployments
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- config.openshift.io
resources:
Expand Down
46 changes: 46 additions & 0 deletions hack/make_fast.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
#!/bin/bash

# Define a function to check OpenShift cluster nodes
check_ocp() {
local KUBECONFIG_PATH=$1

export KUBECONFIG=$KUBECONFIG_PATH

if oc get nodes; then
return 0
else
return 1
fi
}

deploy_cluster_and_dpu_operator() {
bash hack/prepare.sh
bash hack/ipu_host_deploy.sh
bash hack/ipu_deploy.sh
bash hack/deploy_traffic_flow_tests.sh
}

deploy_local_dpu_operator() {
echo "deploying local"
bash hack/prepare.sh
#bash hack/ipu_host_deploy_post.sh
#bash hack/ipu_deploy_post.sh
#bash hack/deploy_traffic_flow_tests.sh

}

check_ocp "/root/kubeconfig.ocpcluster"
result_ocp=$? # Store the return value (0 for true, 1 for false)

check_ocp "/root/kubeconfig.microshift"
result_microshift=$? # Store the return value (0 for true, 1 for false)

# Check if both OCP and MicroShift checks succeeded
if [ $result_ocp -eq 0 ] && [ $result_microshift -eq 0 ]; then
echo "Both OCP and MicroShift cluster checks succeeded!"
deploy_local_dpu_operator
else
echo "Cluster config is not proper... deploying cluster and dpu operator"
#deploy_cluster_and_dpu_operator
fi

13 changes: 13 additions & 0 deletions internal/controller/bindata/webhook/000_service.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
apiVersion: v1
kind: Service
metadata:
name: network-resources-injector-service
namespace: openshift-dpu-operator
annotations:
service.alpha.openshift.io/serving-cert-secret-name: network-resources-injector-secret
spec:
ports:
- port: 443
targetPort: 8443
selector:
app: network-resources-injector
6 changes: 6 additions & 0 deletions internal/controller/bindata/webhook/00_serviceaccount.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
apiVersion: v1
kind: ServiceAccount
metadata:
namespace: openshift-dpu-operator
name: network-resources-injector-sa

Original file line number Diff line number Diff line change
@@ -0,0 +1,27 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: network-resources-injector
rules:
- apiGroups:
- k8s.cni.cncf.io
resources:
- network-attachment-definitions
verbs:
- 'watch'
- 'list'
- 'get'
- apiGroups:
- ""
resources:
- configmaps
verbs:
- 'get'
- apiGroups:
- apps
resources:
- deployments
verbs:
- 'watch'
- 'list'
- 'get'
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: network-resources-injector-role-binding
roleRef:
kind: ClusterRole
name: network-resources-injector
subjects:
- kind: ServiceAccount
name: network-resources-injector-sa
namespace: openshift-dpu-operator
85 changes: 85 additions & 0 deletions internal/controller/bindata/webhook/12-server.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
apiVersion: apps/v1
kind: Deployment
metadata:
labels:
app: network-resources-injector
name: network-resources-injector
namespace: openshift-dpu-operator
spec:
selector:
matchLabels:
app: network-resources-injector
template:
metadata:
labels:
app: network-resources-injector
spec:
serviceAccount: network-resources-injector-sa
containers:
- name: webhook-server
image: quay.io/vpunj/sriov-dp-admission-controller:latest
imagePullPolicy: IfNotPresent
command:
- webhook
args:
- -bind-address=0.0.0.0
- -port=8443
- -tls-private-key-file=/etc/tls/tls.key
- -tls-cert-file=/etc/tls/tls.crt
- -health-check-port=8444
- -logtostderr
env:
- name: NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
securityContext:
runAsUser: 10000
runAsGroup: 10000
capabilities:
drop:
- ALL
add: ["NET_BIND_SERVICE"]
readOnlyRootFilesystem: true
allowPrivilegeEscalation: false
volumeMounts:
- mountPath: /etc/tls
name: tls
resources:
requests:
memory: "50Mi"
cpu: "250m"
limits:
memory: "200Mi"
cpu: "500m"
livenessProbe:
httpGet:
path: /healthz
port: 8444
initialDelaySeconds: 10
periodSeconds: 5
initContainers:
- name: installer
image: quay.io/vpunj/sriov-dp-admission-controller:latest
imagePullPolicy: IfNotPresent
command:
- installer
args:
- -name=network-resources-injector
- -namespace=openshift-dpu-operator
- -alsologtostderr
securityContext:
runAsUser: 10000
runAsGroup: 10000
volumeMounts:
- name: tls
mountPath: /etc/tls
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
volumes:
- name: tls
secret:
secretName: network-resources-injector-secret
28 changes: 28 additions & 0 deletions internal/controller/bindata/webhook/13-webhook.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,28 @@
apiVersion: admissionregistration.k8s.io/v1
kind: MutatingWebhookConfiguration
metadata:
name: network-resources-injector-config
namespace: {{.Namespace}}
annotations:
service.beta.openshift.io/inject-cabundle: "true"
webhooks:
- name: network-resources-injector-config.k8s.io
sideEffects: None
admissionReviewVersions: ["v1", "v1beta1"]
clientConfig:
service:
name: network-resources-injector-service
namespace: {{.Namespace}}
path: "/mutate"
caBundle: {{ .injectorWebhookCA }}
namespaceSelector:
matchExpressions:
- key: "kubernetes.io/metadata.name"
operator: "NotIn"
values:
- "openshift-dpu-operator"
rules:
- operations: [ "CREATE" ]
apiGroups: ["apps", ""]
apiVersions: ["v1"]
resources: ["pods"]
Loading

0 comments on commit 9e90fda

Please sign in to comment.