diff --git a/README.md b/README.md index 0aa68cb35..c1823d489 100644 --- a/README.md +++ b/README.md @@ -117,7 +117,7 @@ The [SGX device plugin](cmd/sgx_plugin/README.md) allows workloads to use Intel® Software Guard Extensions (Intel® SGX) on platforms with SGX Flexible Launch Control enabled, e.g.,: -- 3rd Generation Intel® Xeon® Scalable processor family, code-named “Ice Lake” +- 3rd Generation Intel® Xeon® Scalable processor family, and later - Intel® Xeon® E3 processor - Intel® NUC Kit NUC7CJYH diff --git a/cmd/sgx_plugin/README.md b/cmd/sgx_plugin/README.md index 4bd281110..821f56c52 100644 --- a/cmd/sgx_plugin/README.md +++ b/cmd/sgx_plugin/README.md @@ -19,7 +19,7 @@ Table of Contents The Intel SGX device plugin and related components allow workloads to use Intel SGX on platforms with SGX Flexible Launch Control enabled, e.g.,: -- 3rd/4th Generation Intel® Xeon® Scalable Platforms +- 3rd Generation Intel® Xeon® Scalable Platforms, and later - Intel® Xeon® E3 - Intel® NUC Kit NUC7CJYH @@ -39,10 +39,16 @@ The SGX plugin can take a number of command line arguments, summarised in the fo |:---- |:-------- |:------- | | -enclave-limit | int | the number of containers per worker node allowed to use `/dev/sgx_enclave` device node (default: `20`) | | -provision-limit | int | the number of containers per worker node allowed to use `/dev/sgx_provision` device node (default: `20`) | +| -dcap-infra-resources | bool | a boolean opt-in flag to register special `qe` and `registration` resources for Intel Data Center Attestation Primitive (DCAP) containers (default: `false`) | The plugin also accepts a number of other arguments related to logging. Please use the `-h` option to see the complete list of logging related options. +Note: `qe` and `registration` resources are intended for a very specific use-case: every SGX enabled +node gets only one such resource and they are consumed by a quoting daemon (e.g., `aesmd` or `tdx-qgs`) +and a platform registration tool (e.g., PCK-ID-Retrieval-Tool), respectively. This is done so that +these containers can run without any elevated privileges. + ## Installation The following sections cover how to use the necessary Kubernetes SGX specific diff --git a/cmd/sgx_plugin/sgx_plugin.go b/cmd/sgx_plugin/sgx_plugin.go index cc8cd289a..0513562e1 100644 --- a/cmd/sgx_plugin/sgx_plugin.go +++ b/cmd/sgx_plugin/sgx_plugin.go @@ -25,6 +25,7 @@ import ( dpapi "github.com/intel/intel-device-plugins-for-kubernetes/pkg/deviceplugin" "k8s.io/klog/v2" pluginapi "k8s.io/kubelet/pkg/apis/deviceplugin/v1beta1" + cdispec "tags.cncf.io/container-device-interface/specs-go" ) const ( @@ -38,18 +39,20 @@ const ( ) type devicePlugin struct { - scanDone chan bool - devfsDir string - nEnclave uint - nProvision uint + scanDone chan bool + devfsDir string + nEnclave uint + nProvision uint + dcapInfraResources bool } -func newDevicePlugin(devfsDir string, nEnclave, nProvision uint) *devicePlugin { +func newDevicePlugin(devfsDir string, nEnclave, nProvision uint, dcapInfraResources bool) *devicePlugin { return &devicePlugin{ - devfsDir: devfsDir, - nEnclave: nEnclave, - nProvision: nProvision, - scanDone: make(chan bool, 1), + devfsDir: devfsDir, + nEnclave: nEnclave, + nProvision: nProvision, + dcapInfraResources: dcapInfraResources, + scanDone: make(chan bool, 1), } } @@ -96,6 +99,43 @@ func (dp *devicePlugin) scan() (dpapi.DeviceTree, error) { devTree.AddDevice(deviceTypeProvision, devID, dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, nodes, nil, nil, nil, nil, nil)) } + if !dp.dcapInfraResources { + return devTree, nil + } + + qeNodes := []pluginapi.DeviceSpec{ + {HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}, + {HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}, + } + + devTree.AddDevice("qe", "qe-1", dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, qeNodes, nil, nil, nil, nil, nil)) + + regNodes := []pluginapi.DeviceSpec{ + {HostPath: sgxEnclavePath, ContainerPath: sgxEnclavePath, Permissions: "rw"}, + {HostPath: sgxProvisionPath, ContainerPath: sgxProvisionPath, Permissions: "rw"}, + } + + // /sys/firmware is a maskedPath (see OCI runtime spec.) set by runtimes so /sys/firmware/efi/efivars mount point cannot + // be made visible to containers without running them as privileged. Here, efivarfs gets mounted to /run/efivars as native + // efivarfs fs type (a bind mount would also work) to avoid elevated privileges (NB: efivarfs sets "non-standard" EFI variables + // as "IMMUTABLE" so CAP_LINUX_IMMUTABLE capability is needed for write). Applications must be adapted to the containerPath set here. + efiVarFsMount := &cdispec.Spec{ + Version: dpapi.CDIVersion, + Kind: dpapi.CDIVendor + "/sgx", + Devices: []cdispec.Device{ + { + Name: "efivarfs", + ContainerEdits: cdispec.ContainerEdits{ + Mounts: []*cdispec.Mount{ + {HostPath: "efivarfs", ContainerPath: "/run/efivars", Type: "efivarfs", Options: []string{"rw", "nosuid", "nodev", "noexec", "relatime"}}, + }, + }, + }, + }, + } + + devTree.AddDevice("registration", "registration-1", dpapi.NewDeviceInfoWithTopologyHints(pluginapi.Healthy, regNodes, nil, nil, nil, nil, efiVarFsMount)) + return devTree, nil } @@ -121,15 +161,18 @@ func getDefaultPodCount(nCPUs uint) uint { func main() { var enclaveLimit, provisionLimit uint + var dcapInfraResources bool + podCount := getDefaultPodCount(uint(runtime.NumCPU())) flag.UintVar(&enclaveLimit, "enclave-limit", podCount, "Number of \"enclave\" resources") flag.UintVar(&provisionLimit, "provision-limit", podCount, "Number of \"provision\" resources") + flag.BoolVar(&dcapInfraResources, "dcap-infra-resources", false, "Register special resources for Intel DCAP infrastructure containers") flag.Parse() klog.V(4).Infof("SGX device plugin started with %d \"%s/enclave\" resources and %d \"%s/provision\" resources.", enclaveLimit, namespace, provisionLimit, namespace) - plugin := newDevicePlugin(devicePath, enclaveLimit, provisionLimit) + plugin := newDevicePlugin(devicePath, enclaveLimit, provisionLimit, dcapInfraResources) manager := dpapi.NewManager(namespace, plugin) manager.Run() } diff --git a/cmd/sgx_plugin/sgx_plugin_test.go b/cmd/sgx_plugin/sgx_plugin_test.go index f5a602837..3540f91e5 100644 --- a/cmd/sgx_plugin/sgx_plugin_test.go +++ b/cmd/sgx_plugin/sgx_plugin_test.go @@ -27,17 +27,22 @@ func init() { _ = flag.Set("v", "4") // Enable debug output } +// Update if new resource types are added. +const dcapInfraResources = 2 + // mockNotifier implements Notifier interface. type mockNotifier struct { scanDone chan bool enclaveDevCount int provisionDevCount int + dcapInfraResCnt int } // Notify stops plugin Scan. func (n *mockNotifier) Notify(newDeviceTree dpapi.DeviceTree) { n.enclaveDevCount = len(newDeviceTree[deviceTypeEnclave]) n.provisionDevCount = len(newDeviceTree[deviceTypeProvision]) + n.dcapInfraResCnt = len(newDeviceTree) - n.enclaveDevCount - n.provisionDevCount n.scanDone <- true } @@ -95,6 +100,7 @@ func TestScan(t *testing.T) { requestedProvisionDevs uint expectedEnclaveDevs int expectedProvisionDevs int + requestDcapInfra bool }{ { name: "no device installed", @@ -131,6 +137,16 @@ func TestScan(t *testing.T) { requestedProvisionDevs: 20, expectedProvisionDevs: 20, }, + { + name: "all resources", + enclaveDevice: "sgx_enclave", + provisionDevice: "sgx_provision", + requestedEnclaveDevs: 1, + expectedEnclaveDevs: 1, + requestedProvisionDevs: 1, + expectedProvisionDevs: 1, + requestDcapInfra: true, + }, } for _, tc := range tcases { @@ -159,7 +175,7 @@ func TestScan(t *testing.T) { } } - plugin := newDevicePlugin(devfs, tc.requestedEnclaveDevs, tc.requestedProvisionDevs) + plugin := newDevicePlugin(devfs, tc.requestedEnclaveDevs, tc.requestedProvisionDevs, tc.requestDcapInfra) notifier := &mockNotifier{ scanDone: plugin.scanDone, @@ -175,6 +191,9 @@ func TestScan(t *testing.T) { if tc.expectedProvisionDevs != notifier.provisionDevCount { t.Errorf("Wrong number of discovered provision devices") } + if tc.requestDcapInfra && notifier.dcapInfraResCnt != dcapInfraResources { + t.Errorf("Wrong number of discovered DCAP infra resources: expected %d, got %d.", dcapInfraResources, notifier.dcapInfraResCnt) + } }) } } diff --git a/deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml b/deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml index a50a2354b..b3d933848 100644 --- a/deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml +++ b/deployments/operator/crd/bases/deviceplugin.intel.com_sgxdeviceplugins.yaml @@ -55,6 +55,10 @@ spec: spec: description: SgxDevicePluginSpec defines the desired state of SgxDevicePlugin. properties: + dcapInfraResources: + description: DcapInfraResources flag enables two special resources + for Intel DCAP infrastructure containers. + type: boolean enclaveLimit: description: EnclaveLimit is a number of containers that can share the same SGX enclave device. diff --git a/deployments/sgx_plugin/base/intel-sgx-plugin.yaml b/deployments/sgx_plugin/base/intel-sgx-plugin.yaml index 356650554..597bca7b0 100644 --- a/deployments/sgx_plugin/base/intel-sgx-plugin.yaml +++ b/deployments/sgx_plugin/base/intel-sgx-plugin.yaml @@ -49,6 +49,8 @@ spec: - name: sgx-provision mountPath: /dev/sgx_provision readOnly: true + - name: cdipath + mountPath: /var/run/cdi volumes: - name: kubeletsockets hostPath: @@ -61,5 +63,9 @@ spec: hostPath: path: /dev/sgx_provision type: CharDevice + - name: cdipath + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate nodeSelector: kubernetes.io/arch: amd64 diff --git a/pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go index 1b2bfb6be..c8d407621 100644 --- a/pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go +++ b/pkg/apis/deviceplugin/v1/sgxdeviceplugin_types.go @@ -46,6 +46,9 @@ type SgxDevicePluginSpec struct { // +kubebuilder:validation:Minimum=1 ProvisionLimit int `json:"provisionLimit,omitempty"` + // DcapInfraResources flag enables two special resources for Intel DCAP infrastructure containers. + DcapInfraResources bool `json:"dcapInfraResources,omitempty"` + // LogLevel sets the plugin's log level. // +kubebuilder:validation:Minimum=0 LogLevel int `json:"logLevel,omitempty"` diff --git a/pkg/controllers/sgx/controller.go b/pkg/controllers/sgx/controller.go index d92395870..685876c84 100644 --- a/pkg/controllers/sgx/controller.go +++ b/pkg/controllers/sgx/controller.go @@ -247,5 +247,9 @@ func getPodArgs(sdp *devicepluginv1.SgxDevicePlugin) []string { args = append(args, "-provision-limit", "1") } + if sdp.Spec.DcapInfraResources { + args = append(args, "-dcap-infra-resources") + } + return args } diff --git a/pkg/controllers/sgx/controller_test.go b/pkg/controllers/sgx/controller_test.go index 78c9f4d0f..8c48c7701 100644 --- a/pkg/controllers/sgx/controller_test.go +++ b/pkg/controllers/sgx/controller_test.go @@ -39,6 +39,7 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet yes := true no := false + directoryOrCreate := v1.HostPathDirectoryOrCreate charDevice := v1.HostPathCharDev maxUnavailable := intstr.FromInt(1) maxSurge := intstr.FromInt(0) @@ -116,6 +117,10 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet MountPath: "/dev/sgx_provision", ReadOnly: true, }, + { + Name: "cdipath", + MountPath: "/var/run/cdi", + }, }, }, }, @@ -147,6 +152,15 @@ func (c *controller) newDaemonSetExpected(rawObj client.Object) *apps.DaemonSet }, }, }, + { + Name: "cdipath", + VolumeSource: v1.VolumeSource{ + HostPath: &v1.HostPathVolumeSource{ + Path: "/var/run/cdi", + Type: &directoryOrCreate, + }, + }, + }, }, }, }, diff --git a/test/envtest/sgxdeviceplugin_controller_test.go b/test/envtest/sgxdeviceplugin_controller_test.go index b346d40c8..72827bd8e 100644 --- a/test/envtest/sgxdeviceplugin_controller_test.go +++ b/test/envtest/sgxdeviceplugin_controller_test.go @@ -81,6 +81,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() { updatedLogLevel := 2 updatedEnclaveLimit := 2 updatedProvisionLimit := 2 + updatedDcapInfra := true updatedNodeSelector := map[string]string{"updated-sgx-nodeselector": "true"} fetched.Spec.Image = updatedImage @@ -89,6 +90,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() { fetched.Spec.EnclaveLimit = updatedEnclaveLimit fetched.Spec.ProvisionLimit = updatedProvisionLimit fetched.Spec.NodeSelector = updatedNodeSelector + fetched.Spec.DcapInfraResources = updatedDcapInfra Expect(k8sClient.Update(context.Background(), fetched)).Should(Succeed()) fetchedUpdated := &devicepluginv1.SgxDevicePlugin{} @@ -109,6 +111,7 @@ var _ = Describe("SgxDevicePlugin Controller", func() { strconv.Itoa(updatedEnclaveLimit), "-provision-limit", strconv.Itoa(updatedProvisionLimit), + "-dcap-infra-resources", } Expect(ds.Spec.Template.Spec.Containers[0].Args).Should(ConsistOf(expectArgs))