sgx: set epc limits via NRI annotations

Signed-off-by: Mikko Ylinen <[email protected]>

Author: mythi

deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml

@@ -78,6 +78,11 @@ spec:
                 description: NodeSelector provides a simple way to constrain device
                   plugin pods to nodes with particular labels.
                 type: object
+              nriImage:
+                description: 'NRIImage is a container image with SGX Node Resource
+                  Interface (NRI) plugin executable. Set this value if SGX EPC cgroups
+                  limits enforcement is wanted. TODO: is this a good name?'
+                type: string
               provisionLimit:
                 description: ProvisionLimit is a number of containers that can share
                   the same SGX provision device.

deployments/operator/samples/deviceplugin_v1_sgxdeviceplugin.yaml

@@ -4,6 +4,7 @@ metadata:
   name: sgxdeviceplugin-sample
 spec:
   image: intel/intel-sgx-plugin:0.29.0
+  nriImage: ghcr.io/containers/nri-plugins/nri-sgx-epc:v0.3.2
   enclaveLimit: 110
   provisionLimit: 110
   logLevel: 4

deployments/sgx_plugin/overlays/epc-cgroups/kustomization.yaml

@@ -0,0 +1,7 @@
+resources:
+  - ../../base
+
+patches:
+- path: nri_plugin_patch.yaml
+  target:
+    name: intel-sgx-plugin

deployments/sgx_plugin/overlays/epc-cgroups/nri_plugin_patch.yaml

@@ -0,0 +1,21 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: intel-sgx-plugin
+spec:
+  template:
+    spec:
+      containers:
+      - name: nri-sgx-epc
+        image: ghcr.io/containers/nri-plugins/nri-sgx-epc:unstable
+        securityContext:
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+        imagePullPolicy: IfNotPresent
+        volumeMounts:
+        - name: nrisockets
+          mountPath: /var/run/nri
+      volumes:
+      - name: nrisockets
+        hostPath:
+          path: /var/run/nri

pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go

@@ -38,6 +38,11 @@ type SgxDevicePluginSpec struct {
 	// Specialized nodes (e.g., with accelerators) can be Tainted to make sure unwanted pods are not scheduled on them. Tolerations can be set for the plugin pod to neutralize the Taint.
 	Tolerations []v1.Toleration `json:"tolerations,omitempty"`
 
+	// NRIImage is a container image with SGX Node Resource Interface (NRI) plugin executable. Set
+	// this value if SGX EPC cgroups limits enforcement is wanted.
+	// TODO: is this a good name?
+	NRIImage string `json:"nriImage,omitempty"`
+
 	// EnclaveLimit is a number of containers that can share the same SGX enclave device.
 	// +kubebuilder:validation:Minimum=1
 	EnclaveLimit int `json:"enclaveLimit,omitempty"`

pkg/controllers/sgx/controller.go

@@ -112,6 +112,27 @@ func setInitContainer(spec *v1.PodSpec, imageName string) {
 	addVolumeIfMissing(spec, "nfd-features", "/etc/kubernetes/node-feature-discovery/source.d/", v1.HostPathDirectoryOrCreate)
 }
 
+func setNRIContainer(spec *v1.PodSpec, imageName string) {
+	yes := true
+	no := false
+	spec.Containers = append(spec.Containers, v1.Container{
+		Name:            "nri-sgx-epc",
+		Image:           imageName,
+		ImagePullPolicy: "IfNotPresent",
+		SecurityContext: &v1.SecurityContext{
+			ReadOnlyRootFilesystem:   &yes,
+			AllowPrivilegeEscalation: &no,
+		},
+		VolumeMounts: []v1.VolumeMount{
+			{
+				Name:      "nrisockets",
+				MountPath: "/var/run/nri",
+			},
+		},
+	})
+	addVolumeIfMissing(spec, "nrisockets", "/var/run/nri", v1.HostPathDirectoryOrCreate)
+}
+
 func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	devicePlugin := rawObj.(*devicepluginv1.SgxDevicePlugin)
 
@@ -135,6 +156,10 @@ func (c *controller) NewDaemonSet(rawObj client.Object) *apps.DaemonSet {
 	if devicePlugin.Spec.InitImage != "" {
 		setInitContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.InitImage)
 	}
+	// add the optional NRI plugin container
+	if devicePlugin.Spec.NRIImage != "" {
+		setNRIContainer(&daemonSet.Spec.Template.Spec, devicePlugin.Spec.NRIImage)
+	}
 
 	return daemonSet
 }
@@ -170,6 +195,26 @@ func (c *controller) UpdateDaemonSet(rawObj client.Object, ds *apps.DaemonSet) (
 		updated = true
 	}
 
+	// remove NRI plugin
+	if len(ds.Spec.Template.Spec.Containers) > 1 && dp.Spec.NRIImage == "" {
+		ds.Spec.Template.Spec.Containers = []v1.Container{ds.Spec.Template.Spec.Containers[0]}
+		ds.Spec.Template.Spec.Volumes = removeVolume(ds.Spec.Template.Spec.Volumes, "nrisockets")
+		updated = true
+	}
+
+	// update NRI plugin image
+	if len(ds.Spec.Template.Spec.Containers) > 1 && ds.Spec.Template.Spec.Containers[1].Image != dp.Spec.NRIImage {
+		ds.Spec.Template.Spec.Containers[1].Image = dp.Spec.NRIImage
+		updated = true
+	}
+
+	// add NRI plugin image
+	if len(ds.Spec.Template.Spec.Containers) == 1 && dp.Spec.NRIImage != "" {
+		setNRIContainer(&ds.Spec.Template.Spec, dp.Spec.NRIImage)
+
+		updated = true
+	}
+
 	if len(dp.Spec.NodeSelector) > 0 {
 		if !reflect.DeepEqual(ds.Spec.Template.Spec.NodeSelector, dp.Spec.NodeSelector) {
 			ds.Spec.Template.Spec.NodeSelector = dp.Spec.NodeSelector

pkg/webhooks/sgx/sgx.go

@@ -43,6 +43,7 @@ func (s *Mutator) SetupWebhookWithManager(mgr ctrl.Manager) error {
 }
 
 const (
+	epcLimitKey              = "epc-limit.nri.io/container"
 	namespace                = "sgx.intel.com"
 	encl                     = namespace + "/enclave"
 	epc                      = namespace + "/epc"
@@ -156,6 +157,8 @@ func (s *Mutator) Default(ctx context.Context, obj runtime.Object) error {
 			continue
 		}
 
+		pod.Annotations[fmt.Sprintf("%s.%s", epcLimitKey, container.Name)] = fmt.Sprintf("%d", epcSize)
+
 		totalEpc += epcSize
 
 		// Quote Generation Modes:

test/e2e/sgxadmissionwebhook/sgxaadmissionwebhook.go

@@ -69,6 +69,7 @@ func describe() {
 
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("mutates created pods when the container contains the quote generation libraries", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -79,6 +80,7 @@ func describe() {
 
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("mutates created pods when the container uses aesmd from a side-car container to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -93,6 +95,8 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("2Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1048576"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.aesmd"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("mutates created pods where one container uses host/daemonset aesmd to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -106,6 +110,7 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[0].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("1Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("mutates created pods where three containers use host/daemonset aesmd to generate quotes", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")
@@ -125,6 +130,9 @@ func describe() {
 		gomega.Expect(pod.Spec.Containers[2].Env[0].Value).To(gomega.Equal("1"))
 		ginkgo.By("checking the pod total EPC size annotation is correctly set")
 		gomega.Expect(pod.Annotations["sgx.intel.com/epc"]).To(gomega.Equal("3Mi"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test1"]).To(gomega.Equal("1048576"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test2"]).To(gomega.Equal("1048576"))
+		gomega.Expect(pod.Annotations["epc-limit.nri.io/container.test3"]).To(gomega.Equal("1048576"))
 	})
 	ginkgo.It("checks that Volumes and VolumeMounts are created only once", func(ctx context.Context) {
 		ginkgo.By("submitting the pod")

test/envtest/sgxdeviceplugin_controller_test.go

@@ -39,6 +39,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			spec := devicepluginv1.SgxDevicePluginSpec{
 				Image:        "sgx-testimage",
 				InitImage:    "sgx-testinitimage",
+				NRIImage:     "sgx-testnriimage",
 				NodeSelector: map[string]string{"sgx-nodeselector": "true"},
 			}
 
@@ -78,13 +79,15 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			By("updating SgxDevicePlugin successfully")
 			updatedImage := "updated-sgx-testimage"
 			updatedInitImage := "updated-sgx-testinitimage"
+			updatedNRIImage := "updated-sgx-testnriimage"
 			updatedLogLevel := 2
 			updatedEnclaveLimit := 2
 			updatedProvisionLimit := 2
 			updatedNodeSelector := map[string]string{"updated-sgx-nodeselector": "true"}
 
 			fetched.Spec.Image = updatedImage
 			fetched.Spec.InitImage = updatedInitImage
+			fetched.Spec.NRIImage = updatedNRIImage
 			fetched.Spec.LogLevel = updatedLogLevel
 			fetched.Spec.EnclaveLimit = updatedEnclaveLimit
 			fetched.Spec.ProvisionLimit = updatedProvisionLimit
@@ -114,13 +117,17 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs))
 			Expect(ds.Spec.Template.Spec.Containers[0].Image).Should(Equal(updatedImage))
 			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(1))
+			Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(2))
+			Expect(ds.Spec.Template.Spec.Containers[1].Image).Should(Equal(updatedNRIImage))
 			Expect(ds.Spec.Template.Spec.InitContainers[0].Image).To(Equal(updatedInitImage))
 			Expect(ds.Spec.Template.Spec.NodeSelector).Should(Equal(updatedNodeSelector))
 
 			By("updating SgxDevicePlugin with different values successfully")
 			updatedInitImage = ""
+			updatedNRIImage = ""
 			updatedNodeSelector = map[string]string{}
 			fetched.Spec.InitImage = updatedInitImage
+			fetched.Spec.NRIImage = updatedNRIImage
 			fetched.Spec.NodeSelector = updatedNodeSelector
 
 			Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed())
@@ -130,6 +137,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			err = k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: expectedDsName}, ds)
 			Expect(err).To(BeNil())
 			Expect(ds.Spec.Template.Spec.InitContainers).To(HaveLen(0))
+			Expect(ds.Spec.Template.Spec.Containers).To(HaveLen(1))
 			Expect(ds.Spec.Template.Spec.NodeSelector).Should(And(HaveLen(1), HaveKeyWithValue("kubernetes.io/arch", "amd64")))
 
 			By("updating SgxDevicePlugin with tolerations")