Merge pull request #2103 from mythi/PR-2025-013

sgx: add two special resources for quoting daemon and SGX platform registration

Author: tkatila

README.md

@@ -117,7 +117,7 @@ The [SGX device plugin](cmd/sgx_plugin/README.md) allows workloads to use
 Intel® Software Guard Extensions (Intel® SGX) on
 platforms with SGX Flexible Launch Control enabled, e.g.,:
 
-- 3rd Generation Intel® Xeon® Scalable processor family, code-named “Ice Lake”
+- 3rd Generation Intel® Xeon® Scalable processor family, and later
 - Intel® Xeon® E3 processor
 - Intel® NUC Kit NUC7CJYH
 

cmd/sgx_plugin/README.md

@@ -19,7 +19,7 @@ Table of Contents
 The Intel SGX device plugin and related components allow workloads to use Intel SGX on
 platforms with SGX Flexible Launch Control enabled, e.g.,:
 
-- 3rd/4th Generation Intel® Xeon® Scalable Platforms
+- 3rd Generation Intel® Xeon® Scalable Platforms, and later
 - Intel® Xeon® E3
 - Intel® NUC Kit NUC7CJYH
 
@@ -39,10 +39,16 @@ The SGX plugin can take a number of command line arguments, summarised in the fo
 |:---- |:-------- |:------- |
 | -enclave-limit | int | the number of containers per worker node allowed to use `/dev/sgx_enclave` device node (default: `20`) |
 | -provision-limit | int | the number of containers per worker node allowed to use `/dev/sgx_provision` device node (default: `20`) |
+| -dcap-infra-resources | bool | a boolean opt-in flag to register special `qe` and `registration` resources for Intel Data Center Attestation Primitive (DCAP) containers (default: `false`) |
 
 The plugin also accepts a number of other arguments related to logging. Please use the `-h` option to see
 the complete list of logging related options.
 
+Note: `qe` and `registration` resources are intended for a very specific use-case: every SGX enabled
+node gets only one such resource and they are consumed by a quoting daemon (e.g., `aesmd` or `tdx-qgs`)
+and a platform registration tool (e.g., PCK-ID-Retrieval-Tool), respectively. This is done so that
+these containers can run without any elevated privileges.
+
 ## Installation
 
 The following sections cover how to use the necessary Kubernetes SGX specific

cmd/sgx_plugin/sgx_plugin.go

@@ -25,6 +25,7 @@ import (
 	dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin"
 	"k8s.io/klog/v2"
 	pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1"
+	cdispec "tags.cncf.io/container-device-interface/specs-go"
 )
 
 const (
@@ -38,18 +39,20 @@ const (
 )
 
 type devicePlugin struct {
-	scanDone   chan bool
-	devfsDir   string
-	nEnclave   uint
-	nProvision uint
+	scanDone           chan bool
+	devfsDir           string
+	nEnclave           uint
+	nProvision         uint
+	dcapInfraResources bool
 }
 
-func newDevicePlugin(devfsDir string, nEnclave, nProvision uint) *devicePlugin {
+func newDevicePlugin(devfsDir string, nEnclave, nProvision uint, dcapInfraResources bool) *devicePlugin {
 	return &devicePlugin{
-		devfsDir:   devfsDir,
-		nEnclave:   nEnclave,
-		nProvision: nProvision,
-		scanDone:   make(chan bool, 1),
+		devfsDir:           devfsDir,
+		nEnclave:           nEnclave,
+		nProvision:         nProvision,
+		dcapInfraResources: dcapInfraResources,
+		scanDone:           make(chan bool, 1),
 	}
 }
 
@@ -96,6 +99,43 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) {
 		devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, nodes, nil, nil, nil, nil, nil))
 	}
 
+	if !dp.dcapInfraResources {
+		return devTree, nil
+	}
+
+	qeNodes := []pluginapi.DeviceSpec{
+		{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"},
+		{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"},
+	}
+
+	devTree.AddDevice("qe", "qe-1", dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, qeNodes, nil, nil, nil, nil, nil))
+
+	regNodes := []pluginapi.DeviceSpec{
+		{HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"},
+		{HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"},
+	}
+
+	// /sys/firmware is a maskedPath (see OCI runtime spec.) set by runtimes so /sys/firmware/efi/efivars mount point cannot
+	// be made visible to containers without running them as privileged. Here, efivarfs gets mounted to /run/efivars as native
+	// efivarfs fs type (a bind mount would also work) to avoid elevated privileges (NB: efivarfs sets "non-standard" EFI variables
+	// as "IMMUTABLE" so CAP_LINUX_IMMUTABLE capability is needed for write). Applications must be adapted to the containerPath set here.
+	efiVarFsMount := &cdispec.Spec{
+		Version: dpapi.CDIVersion,
+		Kind:    dpapi.CDIVendor + "/sgx",
+		Devices: []cdispec.Device{
+			{
+				Name: "efivarfs",
+				ContainerEdits: cdispec.ContainerEdits{
+					Mounts: []*cdispec.Mount{
+						{HostPath: "efivarfs", ContainerPath: "/run/efivars", Type: "efivarfs", Options: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}},
+					},
+				},
+			},
+		},
+	}
+
+	devTree.AddDevice("registration", "registration-1", dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, regNodes, nil, nil, nil, nil, efiVarFsMount))
+
 	return devTree, nil
 }
 
@@ -121,15 +161,18 @@ func getDefaultPodCount(nCPUs uint) uint {
 func main() {
 	var enclaveLimit, provisionLimit uint
 
+	var dcapInfraResources bool
+
 	podCount := getDefaultPodCount(uint(runtime.NumCPU()))
 
 	flag.UintVar(&enclaveLimit, "enclave-limit", podCount, "Number of \"enclave\" resources")
 	flag.UintVar(&provisionLimit, "provision-limit", podCount, "Number of \"provision\" resources")
+	flag.BoolVar(&dcapInfraResources, "dcap-infra-resources", false, "Register special resources for Intel DCAP infrastructure containers")
 	flag.Parse()
 
 	klog.V(4).Infof("SGX device plugin started with %d \"%s/enclave\" resources and %d \"%s/provision\" resources.", enclaveLimit, namespace, provisionLimit, namespace)
 
-	plugin := newDevicePlugin(devicePath, enclaveLimit, provisionLimit)
+	plugin := newDevicePlugin(devicePath, enclaveLimit, provisionLimit, dcapInfraResources)
 	manager := dpapi.NewManager(namespace, plugin)
 	manager.Run()
 }

cmd/sgx_plugin/sgx_plugin_test.go

@@ -27,17 +27,22 @@ func init() {
 	_ = flag.Set("v", "4") // Enable debug output
 }
 
+// Update if new resource types are added.
+const dcapInfraResources = 2
+
 // mockNotifier implements Notifier interface.
 type mockNotifier struct {
 	scanDone          chan bool
 	enclaveDevCount   int
 	provisionDevCount int
+	dcapInfraResCnt   int
 }
 
 // Notify stops plugin Scan.
 func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) {
 	n.enclaveDevCount = len(newDeviceTree[deviceTypeEnclave])
 	n.provisionDevCount = len(newDeviceTree[deviceTypeProvision])
+	n.dcapInfraResCnt = len(newDeviceTree) - n.enclaveDevCount - n.provisionDevCount
 	n.scanDone <- true
 }
 
@@ -95,6 +100,7 @@ func TestScan(t *testing.T) {
 		requestedProvisionDevs uint
 		expectedEnclaveDevs    int
 		expectedProvisionDevs  int
+		requestDcapInfra       bool
 	}{
 		{
 			name: "no device installed",
@@ -131,6 +137,16 @@ func TestScan(t *testing.T) {
 			requestedProvisionDevs: 20,
 			expectedProvisionDevs:  20,
 		},
+		{
+			name:                   "all resources",
+			enclaveDevice:          "sgx_enclave",
+			provisionDevice:        "sgx_provision",
+			requestedEnclaveDevs:   1,
+			expectedEnclaveDevs:    1,
+			requestedProvisionDevs: 1,
+			expectedProvisionDevs:  1,
+			requestDcapInfra:       true,
+		},
 	}
 
 	for _, tc := range tcases {
@@ -159,7 +175,7 @@ func TestScan(t *testing.T) {
 				}
 			}
 
-			plugin := newDevicePlugin(devfs, tc.requestedEnclaveDevs, tc.requestedProvisionDevs)
+			plugin := newDevicePlugin(devfs, tc.requestedEnclaveDevs, tc.requestedProvisionDevs, tc.requestDcapInfra)
 
 			notifier := &mockNotifier{
 				scanDone: plugin.scanDone,
@@ -175,6 +191,9 @@ func TestScan(t *testing.T) {
 			if tc.expectedProvisionDevs != notifier.provisionDevCount {
 				t.Errorf("Wrong number of discovered provision devices")
 			}
+			if tc.requestDcapInfra && notifier.dcapInfraResCnt != dcapInfraResources {
+				t.Errorf("Wrong number of discovered DCAP infra resources: expected %d, got %d.", dcapInfraResources, notifier.dcapInfraResCnt)
+			}
 		})
 	}
 }

deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml

@@ -55,6 +55,10 @@ spec:
           spec:
             description: SgxDevicePluginSpec defines the desired state of SgxDevicePlugin.
             properties:
+              dcapInfraResources:
+                description: DcapInfraResources flag enables two special resources
+                  for Intel DCAP infrastructure containers.
+                type: boolean
               enclaveLimit:
                 description: EnclaveLimit is a number of containers that can share
                   the same SGX enclave device.

deployments/sgx_plugin/base/intel-sgx-plugin.yaml

@@ -49,6 +49,8 @@ spec:
         - name: sgx-provision
           mountPath: /dev/sgx_provision
           readOnly: true
+        - name: cdipath
+          mountPath: /var/run/cdi
       volumes:
       - name: kubeletsockets
         hostPath:
@@ -61,5 +63,9 @@ spec:
         hostPath:
           path: /dev/sgx_provision
           type: CharDevice
+      - name: cdipath
+        hostPath:
+          path: /var/run/cdi
+          type: DirectoryOrCreate
       nodeSelector:
         kubernetes.io/arch: amd64

pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go

@@ -46,6 +46,9 @@ type SgxDevicePluginSpec struct {
 	// +kubebuilder:validation:Minimum=1
 	ProvisionLimit int `json:"provisionLimit,omitempty"`
 
+	// DcapInfraResources flag enables two special resources for Intel DCAP infrastructure containers.
+	DcapInfraResources bool `json:"dcapInfraResources,omitempty"`
+
 	// LogLevel sets the plugin's log level.
 	// +kubebuilder:validation:Minimum=0
 	LogLevel int `json:"logLevel,omitempty"`

pkg/controllers/sgx/controller.go

@@ -247,5 +247,9 @@ func getPodArgs(sdp *devicepluginv1.SgxDevicePlugin) []string {
 		args = append(args, "-provision-limit", "1")
 	}
 
+	if sdp.Spec.DcapInfraResources {
+		args = append(args, "-dcap-infra-resources")
+	}
+
 	return args
 }

pkg/controllers/sgx/controller_test.go

@@ -39,6 +39,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 
 	yes := true
 	no := false
+	directoryOrCreate := v1.HostPathDirectoryOrCreate
 	charDevice := v1.HostPathCharDev
 	maxUnavailable := intstr.FromInt(1)
 	maxSurge := intstr.FromInt(0)
@@ -116,6 +117,10 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 									MountPath: "/dev/sgx_provision",
 									ReadOnly:  true,
 								},
+								{
+									Name:      "cdipath",
+									MountPath: "/var/run/cdi",
+								},
 							},
 						},
 					},
@@ -147,6 +152,15 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet
 								},
 							},
 						},
+						{
+							Name: "cdipath",
+							VolumeSource: v1.VolumeSource{
+								HostPath: &v1.HostPathVolumeSource{
+									Path: "/var/run/cdi",
+									Type: &directoryOrCreate,
+								},
+							},
+						},
 					},
 				},
 			},

test/envtest/sgxdeviceplugin_controller_test.go

@@ -81,6 +81,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			updatedLogLevel := 2
 			updatedEnclaveLimit := 2
 			updatedProvisionLimit := 2
+			updatedDcapInfra := true
 			updatedNodeSelector := map[string]string{"updated-sgx-nodeselector": "true"}
 
 			fetched.Spec.Image = updatedImage
@@ -89,6 +90,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 			fetched.Spec.EnclaveLimit = updatedEnclaveLimit
 			fetched.Spec.ProvisionLimit = updatedProvisionLimit
 			fetched.Spec.NodeSelector = updatedNodeSelector
+			fetched.Spec.DcapInfraResources = updatedDcapInfra
 
 			Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed())
 			fetchedUpdated := &devicepluginv1.SgxDevicePlugin{}
@@ -109,6 +111,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() {
 				strconv.Itoa(updatedEnclaveLimit),
 				"-provision-limit",
 				strconv.Itoa(updatedProvisionLimit),
+				"-dcap-infra-resources",
 			}
 
 			Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs))