diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 46d16c81b..12e47fcad 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -48,6 +48,13 @@ jobs: - name: Checkout repository uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + - name: Set up Go + if: matrix.language == 'go' + uses: actions/setup-go@4b73464bb391d4059bd26b0524d20df3927bd417 # v6.3.0 + with: + go-version: '1.23.x' + cache: false + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@68bde559dea0fdcac2102bfdf6230c5f70eb485e # v3.29.5 diff --git a/.github/workflows/generate-catalog.yaml b/.github/workflows/generate-catalog.yaml index 90f6316f8..5e65b7f3c 100644 --- a/.github/workflows/generate-catalog.yaml +++ b/.github/workflows/generate-catalog.yaml @@ -4,10 +4,12 @@ on: branches: - master paths: + - ".github/workflows/generate-catalog.yaml" - "library/**" - "src/**" pull_request: paths: + - ".github/workflows/generate-catalog.yaml" - "library/**" - "src/**" workflow_dispatch: @@ -45,7 +47,7 @@ jobs: - name: Set up Go uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: - go-version: "1.24" + go-version: "1.25.x" check-latest: true - name: Build gator @@ -69,7 +71,7 @@ jobs: --output=catalog.yaml \ --name=gatekeeper-library \ --version=${{ steps.version.outputs.version }} \ - --repository=https://github.com/open-policy-agent/gatekeeper-library + --base-url=https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master - name: Validate catalog run: | @@ -126,7 +128,7 @@ jobs: labels: automation - name: Create PR comment (on PR) - if: github.event_name == 'pull_request' && steps.diff.outputs.changed == 'true' + if: github.event_name == 'pull_request' && steps.diff.outputs.changed == 'true' && github.event.pull_request.head.repo.full_name == github.repository env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | diff --git a/.github/workflows/scripts.yaml b/.github/workflows/scripts.yaml index 7151ad32e..b1fd68060 100644 --- a/.github/workflows/scripts.yaml +++ b/.github/workflows/scripts.yaml @@ -5,6 +5,7 @@ on: - master paths: - ".github/workflows/scripts.yaml" + - "go.work" - "scripts/**" permissions: contents: read @@ -24,7 +25,7 @@ jobs: steps: - uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: - go-version: '1.20' + go-version: '1.23.x' cache: false - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - name: golangci-lint diff --git a/.github/workflows/workflow.yaml b/.github/workflows/workflow.yaml index 796a4ce5d..952c0aaa1 100644 --- a/.github/workflows/workflow.yaml +++ b/.github/workflows/workflow.yaml @@ -65,7 +65,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - gatekeeper: [ "3.20.1", "3.21.0" ] + gatekeeper: [ "3.21.1", "3.22.0" ] engine: [ "cel", "rego" ] name: "Integration test on Gatekeeper ${{ matrix.gatekeeper }} for ${{ matrix.engine }} policies" steps: @@ -86,7 +86,7 @@ jobs: - name: Run integration test run: | - make test-integration + make test-integration POLICY_ENGINE=${{ matrix.engine }} - name: Save logs run: | @@ -131,7 +131,7 @@ jobs: strategy: matrix: engine: [ "cel", "rego" ] - gatekeeper: [ "3.20.1", "3.21.0" ] + gatekeeper: [ "3.21.1", "3.22.0" ] name: "Verify assertions in suite.yaml files for ${{ matrix.engine }} policies" steps: - name: Harden Runner diff --git a/Makefile b/Makefile index 4bfa3dafb..2a153a32c 100755 --- a/Makefile +++ b/Makefile @@ -45,7 +45,7 @@ uninstall: helm uninstall -n gatekeeper-system gatekeeper test-integration: - bats -t test/bats/test.bats + POLICY_ENGINE=$(POLICY_ENGINE) bats -t test/bats/test.bats .PHONY: verify-gator verify-gator: diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/artifacthub-pkg.yml b/artifacthub/library/general/gpuactivedeadline/1.0.0/artifacthub-pkg.yml new file mode 100644 index 000000000..e658ea308 --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/artifacthub-pkg.yml @@ -0,0 +1,22 @@ +version: 1.0.0 +name: k8sgpuactivedeadline +displayName: GPU Active Deadline Required +createdAt: "2026-03-17T00:04:28Z" +description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set activeDeadlineSeconds. This prevents runaway training jobs from holding GPU resources indefinitely. +digest: f15fa92d15ee17101b77cea310b9766253332b9bfcd50447c4487f9eeaef856c +license: Apache-2.0 +homeURL: https://open-policy-agent.github.io/gatekeeper-library/website/gpuactivedeadline +keywords: + - gatekeeper + - open-policy-agent + - policies +readme: |- + # GPU Active Deadline Required + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set activeDeadlineSeconds. This prevents runaway training jobs from holding GPU resources indefinitely. +install: |- + ### Usage + ```shell + kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/artifacthub/library/general/gpuactivedeadline/1.0.0/template.yaml + ``` +provider: + name: Gatekeeper Library diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/kustomization.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-exempt/constraint.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-exempt/constraint.yaml new file mode 100644 index 000000000..43cf7905a --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-exempt/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxActiveDeadlineSeconds: 86400 + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-exempt/example_allowed.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-exempt/example_allowed.yaml new file mode 100644 index 000000000..6b457f9f8 --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-exempt/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/constraint.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/constraint.yaml new file mode 100644 index 000000000..9b2f2b40a --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxActiveDeadlineSeconds: 86400 diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/example_allowed.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/example_allowed.yaml new file mode 100644 index 000000000..ee9a81aea --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-with-deadline +spec: + activeDeadlineSeconds: 3600 + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml new file mode 100644 index 000000000..eeb36103b --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-exceeds-deadline +spec: + activeDeadlineSeconds: 172800 + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-without-deadline/constraint.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-without-deadline/constraint.yaml new file mode 100644 index 000000000..71df2680f --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-without-deadline/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-without-deadline/example_disallowed.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-without-deadline/example_disallowed.yaml new file mode 100644 index 000000000..11a08f60b --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/gpu-job-without-deadline/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-without-deadline +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/non-gpu-job/constraint.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/non-gpu-job/constraint.yaml new file mode 100644 index 000000000..71df2680f --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/non-gpu-job/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/non-gpu-job/example_allowed.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/non-gpu-job/example_allowed.yaml new file mode 100644 index 000000000..0c7634b44 --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/samples/non-gpu-job/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-job +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/suite.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/suite.yaml new file mode 100644 index 000000000..d9c338124 --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/suite.yaml @@ -0,0 +1,41 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpuactivedeadline +tests: +- name: gpu-job-with-deadline + template: template.yaml + constraint: samples/gpu-job-with-deadline/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-job-with-deadline/example_allowed.yaml + assertions: + - violations: no + - name: example-disallowed-exceeds-max + object: samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml + assertions: + - violations: yes +- name: gpu-job-without-deadline + template: template.yaml + constraint: samples/gpu-job-without-deadline/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-job-without-deadline/example_disallowed.yaml + assertions: + - violations: yes +- name: non-gpu-job + template: template.yaml + constraint: samples/non-gpu-job/constraint.yaml + cases: + - name: example-allowed + object: samples/non-gpu-job/example_allowed.yaml + assertions: + - violations: no +- name: gpu-job-exempt + template: template.yaml + constraint: samples/gpu-job-exempt/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-job-exempt/example_allowed.yaml + assertions: + - violations: no diff --git a/artifacthub/library/general/gpuactivedeadline/1.0.0/template.yaml b/artifacthub/library/general/gpuactivedeadline/1.0.0/template.yaml new file mode 100644 index 000000000..677d2134b --- /dev/null +++ b/artifacthub/library/general/gpuactivedeadline/1.0.0/template.yaml @@ -0,0 +1,145 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuactivedeadline + annotations: + metadata.gatekeeper.sh/title: "GPU Active Deadline Required" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-ai-training-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set + activeDeadlineSeconds. This prevents runaway training jobs from holding + GPU resources indefinitely. +spec: + crd: + spec: + names: + kind: K8sGpuActiveDeadline + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to set activeDeadlineSeconds. + properties: + maxActiveDeadlineSeconds: + description: >- + The maximum value allowed for activeDeadlineSeconds. Set to 0 to + only require the field is present without enforcing a maximum. + type: integer + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: hasDeadline + expression: 'has(variables.anyObject.spec.activeDeadlineSeconds)' + - name: maxDeadline + expression: 'has(variables.params.maxActiveDeadlineSeconds) ? variables.params.maxActiveDeadlineSeconds : 0' + validations: + - expression: '!variables.podRequestsGpu || variables.hasDeadline' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not set activeDeadlineSeconds"' + - expression: '!variables.podRequestsGpu || !variables.hasDeadline || variables.maxDeadline == 0 || variables.anyObject.spec.activeDeadlineSeconds <= variables.maxDeadline' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> sets activeDeadlineSeconds to " + string(variables.anyObject.spec.activeDeadlineSeconds) + ", which exceeds the maximum allowed " + string(variables.maxDeadline)' + - engine: Rego + source: + rego: | + package k8sgpuactivedeadline + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + not has_active_deadline + msg := sprintf("Pod <%v> requests GPU resources but does not set activeDeadlineSeconds", [input.review.object.metadata.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + has_active_deadline + max_deadline := object.get(input, ["parameters", "maxActiveDeadlineSeconds"], 0) + max_deadline > 0 + deadline := input.review.object.spec.activeDeadlineSeconds + deadline > max_deadline + msg := sprintf("Pod <%v> sets activeDeadlineSeconds to %v, which exceeds the maximum allowed %v", [input.review.object.metadata.name, deadline, max_deadline]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_active_deadline { + input.review.object.spec.activeDeadlineSeconds + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/artifacthub-pkg.yml b/artifacthub/library/general/gpunodetargeting/1.0.0/artifacthub-pkg.yml new file mode 100644 index 000000000..1282ae78b --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/artifacthub-pkg.yml @@ -0,0 +1,22 @@ +version: 1.0.0 +name: k8sgpunodetargeting +displayName: GPU Node Targeting +createdAt: "2026-04-10T20:39:17Z" +description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to target GPU-labeled nodes using required node affinity or nodeSelector. This helps ensure GPU workloads only land on nodes that advertise GPU capacity. +digest: d9fbeda4d7ad9dc93c6c753400e7490154862b11a34d606a9e01cc457e27d7f0 +license: Apache-2.0 +homeURL: https://open-policy-agent.github.io/gatekeeper-library/website/gpunodetargeting +keywords: + - gatekeeper + - open-policy-agent + - policies +readme: |- + # GPU Node Targeting + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to target GPU-labeled nodes using required node affinity or nodeSelector. This helps ensure GPU workloads only land on nodes that advertise GPU capacity. +install: |- + ### Usage + ```shell + kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/artifacthub/library/general/gpunodetargeting/1.0.0/template.yaml + ``` +provider: + name: Gatekeeper Library diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/kustomization.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/kustomization.yaml new file mode 100644 index 000000000..24dedaea1 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-exempt/constraint.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-exempt/constraint.yaml new file mode 100644 index 000000000..37f2a0d77 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-exempt/constraint.yaml @@ -0,0 +1,15 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-exempt/example_allowed.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-exempt/example_allowed.yaml new file mode 100644 index 000000000..9f8e7c5af --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-exempt/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/constraint.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/constraint.yaml new file mode 100644 index 000000000..5ee41f321 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/example_allowed.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/example_allowed.yaml new file mode 100644 index 000000000..001651b65 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/example_allowed.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-node-selector-key-only +spec: + nodeSelector: + nvidia.com/gpu.present: "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml new file mode 100644 index 000000000..dfbbf24b9 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-node-selector-empty-value +spec: + nodeSelector: + nvidia.com/gpu.present: "" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/constraint.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/constraint.yaml new file mode 100644 index 000000000..9f299ce0e --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_allowed.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_allowed.yaml new file mode 100644 index 000000000..9ecc45dea --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_allowed.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml new file mode 100644 index 000000000..9b6121ac5 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-mixed-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + - "false" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml new file mode 100644 index 000000000..6db6bdee4 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-wrong-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "false" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-selector/constraint.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-selector/constraint.yaml new file mode 100644 index 000000000..9f299ce0e --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-selector/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-selector/example_allowed.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-selector/example_allowed.yaml new file mode 100644 index 000000000..3c74a2859 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-with-node-selector/example_allowed.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-node-selector +spec: + nodeSelector: + nvidia.com/gpu.present: "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-without-targeting/constraint.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-without-targeting/constraint.yaml new file mode 100644 index 000000000..9f299ce0e --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-without-targeting/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-without-targeting/example_disallowed.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-without-targeting/example_disallowed.yaml new file mode 100644 index 000000000..c02e06e6d --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/gpu-pod-without-targeting/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-targeting +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/non-gpu-pod/constraint.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/non-gpu-pod/constraint.yaml new file mode 100644 index 000000000..9f299ce0e --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/non-gpu-pod/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/samples/non-gpu-pod/example_allowed.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/non-gpu-pod/example_allowed.yaml new file mode 100644 index 000000000..492a70fdf --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/samples/non-gpu-pod/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/suite.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/suite.yaml new file mode 100644 index 000000000..69dd8420d --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/suite.yaml @@ -0,0 +1,65 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpunodetargeting +tests: +- name: gpu-pod-with-node-affinity + template: template.yaml + constraint: samples/gpu-pod-with-node-affinity/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-with-node-affinity/example_allowed.yaml + assertions: + - violations: no + - name: example-disallowed-wrong-value + object: samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml + assertions: + - violations: yes + - name: example-disallowed-mixed-values + object: samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml + assertions: + - violations: yes +- name: gpu-pod-with-node-selector + template: template.yaml + constraint: samples/gpu-pod-with-node-selector/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-with-node-selector/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-without-targeting + template: template.yaml + constraint: samples/gpu-pod-without-targeting/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-without-targeting/example_disallowed.yaml + assertions: + - violations: yes +- name: non-gpu-pod + template: template.yaml + constraint: samples/non-gpu-pod/constraint.yaml + cases: + - name: example-allowed + object: samples/non-gpu-pod/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-node-selector-key-only + template: template.yaml + constraint: samples/gpu-pod-node-selector-key-only/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-node-selector-key-only/example_allowed.yaml + assertions: + - violations: no + - name: example-disallowed-empty-value + object: samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml + assertions: + - violations: yes +- name: gpu-pod-exempt + template: template.yaml + constraint: samples/gpu-pod-exempt/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-exempt/example_allowed.yaml + assertions: + - violations: no \ No newline at end of file diff --git a/artifacthub/library/general/gpunodetargeting/1.0.0/template.yaml b/artifacthub/library/general/gpunodetargeting/1.0.0/template.yaml new file mode 100644 index 000000000..2019d5ea5 --- /dev/null +++ b/artifacthub/library/general/gpunodetargeting/1.0.0/template.yaml @@ -0,0 +1,245 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpunodetargeting + annotations: + metadata.gatekeeper.sh/title: "GPU Node Targeting" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to target + GPU-labeled nodes using required node affinity or nodeSelector. This helps + ensure GPU workloads only land on nodes that advertise GPU capacity. +spec: + crd: + spec: + names: + kind: K8sGpuNodeTargeting + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to target nodes with a configured GPU label key and optional values. + properties: + nodeLabelKey: + description: >- + The node label key that GPU workloads must target (for example, `nvidia.com/gpu.present` + or `nvidia.com/gpu.product`). + type: string + nodeLabelValues: + description: >- + Optional allowed values for the GPU node label. If omitted, the policy only requires the + label key to be present. + type: array + items: + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: allContainers + expression: 'variables.containers + variables.initContainers + variables.ephemeralContainers' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.allContainers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + variables.allContainers.exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + ( + (has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) || + (has(container.resources.requests) && + "nvidia.com/gpu" in container.resources.requests && + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) + ) + ) + - name: nodeLabelKey + expression: 'has(variables.params.nodeLabelKey) ? variables.params.nodeLabelKey : ""' + - name: nodeLabelValues + expression: 'has(variables.params.nodeLabelValues) ? variables.params.nodeLabelValues : []' + - name: hasMatchingNodeSelector + expression: | + !has(variables.anyObject.spec.nodeSelector) || !(variables.nodeLabelKey in variables.anyObject.spec.nodeSelector) ? false : + (size(variables.nodeLabelValues) == 0 ? + string(variables.anyObject.spec.nodeSelector[variables.nodeLabelKey]) != "" : + variables.anyObject.spec.nodeSelector[variables.nodeLabelKey] in variables.nodeLabelValues) + - name: hasMatchingNodeAffinity + expression: | + !has(variables.anyObject.spec.affinity) || + !has(variables.anyObject.spec.affinity.nodeAffinity) || + !has(variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution) || + !has(variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms) ? false : + variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms.exists(term, + has(term.matchExpressions) && + term.matchExpressions.exists(expr, + expr.key == variables.nodeLabelKey && + ( + size(variables.nodeLabelValues) == 0 ? + expr.operator == "Exists" : + expr.operator == "In" && + has(expr.values) && + size(expr.values) > 0 && + expr.values.all(exprValue, exprValue in variables.nodeLabelValues) + ) + ) + ) + validations: + - expression: '!variables.podRequestsGpu || variables.nodeLabelKey == "" || variables.hasMatchingNodeSelector || variables.hasMatchingNodeAffinity' + messageExpression: | + size(variables.nodeLabelValues) == 0 ? + "Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not target nodes with label key <" + variables.nodeLabelKey + "> using node affinity or nodeSelector" : + "Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not target nodes with label <" + variables.nodeLabelKey + "> matching one of <" + variables.nodeLabelValues.join(", ") + "> using node affinity or nodeSelector" + - engine: Rego + source: + rego: | + package k8sgpunodetargeting + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + label_key := object.get(input.parameters, "nodeLabelKey", "") + label_key != "" + not has_matching_node_selector(label_key) + not has_matching_node_affinity(label_key) + label_values := object.get(input.parameters, "nodeLabelValues", []) + msg := violation_message(label_key, label_values) + } + + violation_message(label_key, label_values) = msg { + count(label_values) == 0 + msg := sprintf("Pod <%v> requests GPU resources but does not target nodes with label key <%v> using node affinity or nodeSelector", [input.review.object.metadata.name, label_key]) + } + + violation_message(label_key, label_values) = msg { + count(label_values) > 0 + msg := sprintf("Pod <%v> requests GPU resources but does not target nodes with label <%v> matching one of <%v> using node affinity or nodeSelector", [input.review.object.metadata.name, label_key, label_values]) + } + + pod_requests_gpu { + container := all_containers[_] + not is_exempt(container) + requests_gpu(container) + } + + all_containers[c] { + c := input.review.object.spec.containers[_] + } + + all_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + all_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + + requests_gpu(container) { + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu := limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + requests_gpu(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + gpu := requests["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_matching_node_selector(label_key) { + selector := input.review.object.spec.nodeSelector + value := selector[label_key] + value != "" + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) == 0 + } + + has_matching_node_selector(label_key) { + selector := input.review.object.spec.nodeSelector + value := selector[label_key] + label_values := object.get(input.parameters, "nodeLabelValues", []) + label_values[_] == value + } + + has_matching_node_affinity(label_key) { + term := input.review.object.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[_] + expr := term.matchExpressions[_] + expr.key == label_key + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) == 0 + expr.operator == "Exists" + } + + has_matching_node_affinity(label_key) { + term := input.review.object.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[_] + expr := term.matchExpressions[_] + expr.key == label_key + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) > 0 + expr.operator == "In" + values := object.get(expr, "values", []) + count(values) > 0 + not has_disallowed_affinity_value(values, label_values) + } + + has_disallowed_affinity_value(values, label_values) { + value := values[_] + not allowed_affinity_value(value, label_values) + } + + allowed_affinity_value(value, label_values) { + label_values[_] == value + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/artifacthub-pkg.yml b/artifacthub/library/general/gpuresourcelimits/1.0.0/artifacthub-pkg.yml new file mode 100644 index 000000000..3ed0c151a --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/artifacthub-pkg.yml @@ -0,0 +1,22 @@ +version: 1.0.0 +name: k8sgpuresourcelimits +displayName: GPU Resource Limits +createdAt: "2026-03-17T00:04:28Z" +description: Enforces a maximum number of NVIDIA GPUs (nvidia.com/gpu) that a single container may request. This prevents individual containers from hoarding GPU resources on shared clusters, particularly for AI/ML training workloads. +digest: cc06ded085ae8ba815f0f1b22884d714d2989dd70cbb4ec4c09334d854cc51eb +license: Apache-2.0 +homeURL: https://open-policy-agent.github.io/gatekeeper-library/website/gpuresourcelimits +keywords: + - gatekeeper + - open-policy-agent + - policies +readme: |- + # GPU Resource Limits + Enforces a maximum number of NVIDIA GPUs (nvidia.com/gpu) that a single container may request. This prevents individual containers from hoarding GPU resources on shared clusters, particularly for AI/ML training workloads. +install: |- + ### Usage + ```shell + kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/artifacthub/library/general/gpuresourcelimits/1.0.0/template.yaml + ``` +provider: + name: Gatekeeper Library diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/kustomization.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exceeds-limit/constraint.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exceeds-limit/constraint.yaml new file mode 100644 index 000000000..73fd79687 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exceeds-limit/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exceeds-limit/example_disallowed.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exceeds-limit/example_disallowed.yaml new file mode 100644 index 000000000..956b583ff --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exceeds-limit/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exceeds-limit +spec: + containers: + - name: training + image: myrepo/large-training:v1 + resources: + limits: + nvidia.com/gpu: "8" diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exempt-over-limit/constraint.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exempt-over-limit/constraint.yaml new file mode 100644 index 000000000..6e87e1ef6 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exempt-over-limit/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exempt-over-limit/example_allowed.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exempt-over-limit/example_allowed.yaml new file mode 100644 index 000000000..f25579b89 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-exempt-over-limit/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-over-limit +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "8" \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-max-disabled/constraint.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-max-disabled/constraint.yaml new file mode 100644 index 000000000..654cb75e8 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-max-disabled/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-max-disabled/example_allowed.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-max-disabled/example_allowed.yaml new file mode 100644 index 000000000..0d397154a --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-max-disabled/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-max-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "8" \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-within-limit/constraint.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-within-limit/constraint.yaml new file mode 100644 index 000000000..73fd79687 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-within-limit/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-within-limit/example_allowed.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-within-limit/example_allowed.yaml new file mode 100644 index 000000000..37809a15b --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/gpu-within-limit/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-within-limit +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "2" diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/init-gpu-exceeds-limit/constraint.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/init-gpu-exceeds-limit/constraint.yaml new file mode 100644 index 000000000..fbb00baff --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/init-gpu-exceeds-limit/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/init-gpu-exceeds-limit/example_disallowed.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/init-gpu-exceeds-limit/example_disallowed.yaml new file mode 100644 index 000000000..5166b5320 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/init-gpu-exceeds-limit/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: init-gpu-exceeds-limit +spec: + initContainers: + - name: setup + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "8" + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/non-gpu-pod/constraint.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/non-gpu-pod/constraint.yaml new file mode 100644 index 000000000..fbb00baff --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/non-gpu-pod/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/non-gpu-pod/example_allowed.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/non-gpu-pod/example_allowed.yaml new file mode 100644 index 000000000..492a70fdf --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/samples/non-gpu-pod/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" \ No newline at end of file diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/suite.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/suite.yaml new file mode 100644 index 000000000..bf69ef012 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/suite.yaml @@ -0,0 +1,53 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpuresourcelimits +tests: +- name: gpu-within-limit + template: template.yaml + constraint: samples/gpu-within-limit/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-within-limit/example_allowed.yaml + assertions: + - violations: no +- name: gpu-exceeds-limit + template: template.yaml + constraint: samples/gpu-exceeds-limit/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-exceeds-limit/example_disallowed.yaml + assertions: + - violations: yes +- name: non-gpu-pod + template: template.yaml + constraint: samples/non-gpu-pod/constraint.yaml + cases: + - name: example-allowed + object: samples/non-gpu-pod/example_allowed.yaml + assertions: + - violations: no +- name: gpu-exempt-over-limit + template: template.yaml + constraint: samples/gpu-exempt-over-limit/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exempt-over-limit/example_allowed.yaml + assertions: + - violations: no +- name: gpu-max-disabled + template: template.yaml + constraint: samples/gpu-max-disabled/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-max-disabled/example_allowed.yaml + assertions: + - violations: no +- name: init-gpu-exceeds-limit + template: template.yaml + constraint: samples/init-gpu-exceeds-limit/constraint.yaml + cases: + - name: example-disallowed + object: samples/init-gpu-exceeds-limit/example_disallowed.yaml + assertions: + - violations: yes diff --git a/artifacthub/library/general/gpuresourcelimits/1.0.0/template.yaml b/artifacthub/library/general/gpuresourcelimits/1.0.0/template.yaml new file mode 100644 index 000000000..299552859 --- /dev/null +++ b/artifacthub/library/general/gpuresourcelimits/1.0.0/template.yaml @@ -0,0 +1,126 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuresourcelimits + annotations: + metadata.gatekeeper.sh/title: "GPU Resource Limits" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Enforces a maximum number of NVIDIA GPUs (nvidia.com/gpu) that a single + container may request. This prevents individual containers from hoarding + GPU resources on shared clusters, particularly for AI/ML training workloads. +spec: + crd: + spec: + names: + kind: K8sGpuResourceLimits + validation: + openAPIV3Schema: + type: object + description: >- + Enforces a maximum number of NVIDIA GPUs per container. + properties: + maxGpuPerContainer: + description: >- + The maximum number of GPUs a single container may request. + type: integer + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: maxGpu + expression: 'has(variables.params.maxGpuPerContainer) ? variables.params.maxGpuPerContainer : 0' + - name: badContainers + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + variables.maxGpu > 0 && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity(string(variables.maxGpu))) > 0 + ).map(container, "Container <" + container.name + "> requests " + string(container.resources.limits["nvidia.com/gpu"]) + " GPUs, which exceeds the maximum allowed " + string(variables.maxGpu)) + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpuresourcelimits + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input_containers[_] + not is_exempt(container) + gpu_count := to_number(container.resources.limits["nvidia.com/gpu"]) + gpu_count > 0 + max_gpu := object.get(input, ["parameters", "maxGpuPerContainer"], 0) + max_gpu > 0 + gpu_count > max_gpu + msg := sprintf("Container <%v> requests %v GPUs, which exceeds the maximum allowed %v", [container.name, gpu_count, max_gpu]) + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/artifacthub-pkg.yml b/artifacthub/library/general/gpusharedmemory/1.0.0/artifacthub-pkg.yml new file mode 100644 index 000000000..f283a51d1 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/artifacthub-pkg.yml @@ -0,0 +1,22 @@ +version: 1.0.0 +name: k8sgpusharedmemory +displayName: GPU Shared Memory Required +createdAt: "2026-03-17T00:04:28Z" +description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to mount a memory-backed emptyDir volume at /dev/shm. PyTorch DataLoader, NCCL multi-GPU communication, and most training frameworks require shared memory beyond the default 64MB. +digest: 21bee1aca71d12b2fb583f1a5e90575558e13fc1be0ad750c281dd3c7f03ae15 +license: Apache-2.0 +homeURL: https://open-policy-agent.github.io/gatekeeper-library/website/gpusharedmemory +keywords: + - gatekeeper + - open-policy-agent + - policies +readme: |- + # GPU Shared Memory Required + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to mount a memory-backed emptyDir volume at /dev/shm. PyTorch DataLoader, NCCL multi-GPU communication, and most training frameworks require shared memory beyond the default 64MB. +install: |- + ### Usage + ```shell + kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/artifacthub/library/general/gpusharedmemory/1.0.0/template.yaml + ``` +provider: + name: Gatekeeper Library diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/kustomization.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-exempt-without-shm/constraint.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-exempt-without-shm/constraint.yaml new file mode 100644 index 000000000..498b77964 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-exempt-without-shm/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-exempt-without-shm/example_allowed.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-exempt-without-shm/example_allowed.yaml new file mode 100644 index 000000000..d839d7e0d --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-exempt-without-shm/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-shm +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-non-memory/constraint.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-non-memory/constraint.yaml new file mode 100644 index 000000000..f8d2ad123 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-non-memory/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-non-memory/example_disallowed.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-non-memory/example_disallowed.yaml new file mode 100644 index 000000000..b3a244b79 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-non-memory/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-shm-non-memory +spec: + volumes: + - name: dshm + emptyDir: {} + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /dev/shm \ No newline at end of file diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-wrong-path/constraint.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-wrong-path/constraint.yaml new file mode 100644 index 000000000..f8d2ad123 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-wrong-path/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-wrong-path/example_disallowed.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-wrong-path/example_disallowed.yaml new file mode 100644 index 000000000..5a09f03b2 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-shm-wrong-path/example_disallowed.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-shm-wrong-path +spec: + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /cache \ No newline at end of file diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-with-shm/constraint.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-with-shm/constraint.yaml new file mode 100644 index 000000000..3ebae67a8 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-with-shm/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-with-shm/example_allowed.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-with-shm/example_allowed.yaml new file mode 100644 index 000000000..978bdeeab --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-with-shm/example_allowed.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-shm +spec: + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 8Gi + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /dev/shm diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-without-shm/constraint.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-without-shm/constraint.yaml new file mode 100644 index 000000000..3ebae67a8 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-without-shm/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-without-shm/example_disallowed.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-without-shm/example_disallowed.yaml new file mode 100644 index 000000000..014e92d27 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/gpu-without-shm/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-shm +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/no-gpu/constraint.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/no-gpu/constraint.yaml new file mode 100644 index 000000000..3ebae67a8 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/no-gpu/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/samples/no-gpu/example_allowed.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/no-gpu/example_allowed.yaml new file mode 100644 index 000000000..2fc1db2ad --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/samples/no-gpu/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/suite.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/suite.yaml new file mode 100644 index 000000000..f30b1cadf --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/suite.yaml @@ -0,0 +1,53 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpusharedmemory +tests: +- name: gpu-with-shm + template: template.yaml + constraint: samples/gpu-with-shm/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-with-shm/example_allowed.yaml + assertions: + - violations: no +- name: gpu-without-shm + template: template.yaml + constraint: samples/gpu-without-shm/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-without-shm/example_disallowed.yaml + assertions: + - violations: yes +- name: no-gpu + template: template.yaml + constraint: samples/no-gpu/constraint.yaml + cases: + - name: example-allowed + object: samples/no-gpu/example_allowed.yaml + assertions: + - violations: no +- name: gpu-shm-non-memory + template: template.yaml + constraint: samples/gpu-shm-non-memory/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-shm-non-memory/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-shm-wrong-path + template: template.yaml + constraint: samples/gpu-shm-wrong-path/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-shm-wrong-path/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-exempt-without-shm + template: template.yaml + constraint: samples/gpu-exempt-without-shm/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exempt-without-shm/example_allowed.yaml + assertions: + - violations: no diff --git a/artifacthub/library/general/gpusharedmemory/1.0.0/template.yaml b/artifacthub/library/general/gpusharedmemory/1.0.0/template.yaml new file mode 100644 index 000000000..a6cc2b545 --- /dev/null +++ b/artifacthub/library/general/gpusharedmemory/1.0.0/template.yaml @@ -0,0 +1,126 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpusharedmemory + annotations: + metadata.gatekeeper.sh/title: "GPU Shared Memory Required" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-ai-training-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to mount a + memory-backed emptyDir volume at /dev/shm. PyTorch DataLoader, NCCL + multi-GPU communication, and most training frameworks require shared memory + beyond the default 64MB. +spec: + crd: + spec: + names: + kind: K8sGpuSharedMemory + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to mount a memory-backed volume at /dev/shm. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.containers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: volumes + expression: 'has(variables.anyObject.spec.volumes) ? variables.anyObject.spec.volumes : []' + - name: memoryVolNames + expression: | + variables.volumes.filter(v, + has(v.emptyDir) && has(v.emptyDir.medium) && v.emptyDir.medium == "Memory" + ).map(v, v.name) + - name: badContainers + expression: | + variables.containers.filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + (!has(container.volumeMounts) || + !container.volumeMounts.exists(vm, + vm.mountPath == "/dev/shm" && + vm.name in variables.memoryVolNames + ) + ) + ).map(container, "Container <" + container.name + "> requests GPU resources but does not mount a memory-backed volume at /dev/shm") + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpusharedmemory + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input.review.object.spec.containers[_] + not is_exempt(container) + has_gpu_request(container) + not has_shm_mount(container) + msg := sprintf("Container <%v> requests GPU resources but does not mount a memory-backed volume at /dev/shm", [container.name]) + } + + has_gpu_request(container) { + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_shm_mount(container) { + mount := container.volumeMounts[_] + mount.mountPath == "/dev/shm" + volume := input.review.object.spec.volumes[_] + volume.name == mount.name + volume.emptyDir.medium == "Memory" + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/artifacthub-pkg.yml b/artifacthub/library/general/gpuworkloadresources/1.0.0/artifacthub-pkg.yml new file mode 100644 index 000000000..88eaa43e4 --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/artifacthub-pkg.yml @@ -0,0 +1,22 @@ +version: 1.0.0 +name: k8sgpuworkloadresources +displayName: GPU Workload Resources +createdAt: "2026-04-10T20:39:17Z" +description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set memory requests equal to limits for all non-exempt containers and to set a CPU request. Containers that request GPUs must also set matching GPU requests and limits. This keeps GPU workloads schedulable and predictable while still allowing CPU burst above request. +digest: 3e1179b7c9c0aecef6cc29593eb3a9bd88356490a64859ab4f0c2bf1eaf7bfbc +license: Apache-2.0 +homeURL: https://open-policy-agent.github.io/gatekeeper-library/website/gpuworkloadresources +keywords: + - gatekeeper + - open-policy-agent + - policies +readme: |- + # GPU Workload Resources + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set memory requests equal to limits for all non-exempt containers and to set a CPU request. Containers that request GPUs must also set matching GPU requests and limits. This keeps GPU workloads schedulable and predictable while still allowing CPU burst above request. +install: |- + ### Usage + ```shell + kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/artifacthub/library/general/gpuworkloadresources/1.0.0/template.yaml + ``` +provider: + name: Gatekeeper Library diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/kustomization.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/kustomization.yaml new file mode 100644 index 000000000..24dedaea1 --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-compliant/constraint.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-compliant/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-compliant/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-compliant/example_allowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-compliant/example_allowed.yaml new file mode 100644 index 000000000..ff4ab90fd --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-compliant/example_allowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-compliant +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-cpu-request-missing/constraint.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-cpu-request-missing/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-cpu-request-missing/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-cpu-request-missing/example_disallowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-cpu-request-missing/example_disallowed.yaml new file mode 100644 index 000000000..b08908bfa --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-cpu-request-missing/example_disallowed.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-cpu-request-missing +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" + - name: logger + image: busybox:1.36 + resources: + requests: + memory: "256Mi" + limits: + memory: "256Mi" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-exempt/constraint.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-exempt/constraint.yaml new file mode 100644 index 000000000..4fdae6b96 --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-exempt/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-exempt/example_allowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-exempt/example_allowed.yaml new file mode 100644 index 000000000..9f8e7c5af --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-exempt/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-mismatch/constraint.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-mismatch/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-mismatch/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-mismatch/example_disallowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-mismatch/example_disallowed.yaml new file mode 100644 index 000000000..a9a0cf9b6 --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-mismatch/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-gpu-mismatch +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "2" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/constraint.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/example_disallowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/example_disallowed.yaml new file mode 100644 index 000000000..3ee8d1b6d --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/example_disallowed.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-gpu-request-missing +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/example_gator_disallowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/example_gator_disallowed.yaml new file mode 100644 index 000000000..3ee8d1b6d --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-gpu-request-missing/example_gator_disallowed.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-gpu-request-missing +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-equivalent/constraint.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-equivalent/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-equivalent/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-equivalent/example_allowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-equivalent/example_allowed.yaml new file mode 100644 index 000000000..5fe5a904d --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-equivalent/example_allowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-memory-equivalent +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "1024Mi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "1Gi" + cpu: "4" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-mismatch/constraint.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-mismatch/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-mismatch/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-mismatch/example_disallowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-mismatch/example_disallowed.yaml new file mode 100644 index 000000000..a513a8bc8 --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/gpu-pod-memory-mismatch/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-memory-mismatch +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "8Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/non-gpu-pod/constraint.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/non-gpu-pod/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/non-gpu-pod/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/non-gpu-pod/example_allowed.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/non-gpu-pod/example_allowed.yaml new file mode 100644 index 000000000..fdf3a64ba --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/samples/non-gpu-pod/example_allowed.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/suite.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/suite.yaml new file mode 100644 index 000000000..0173bd0d0 --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/suite.yaml @@ -0,0 +1,69 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpuworkloadresources +tests: +- name: gpu-pod-compliant + template: template.yaml + constraint: samples/gpu-pod-compliant/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-compliant/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-memory-mismatch + template: template.yaml + constraint: samples/gpu-pod-memory-mismatch/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-memory-mismatch/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-pod-cpu-request-missing + template: template.yaml + constraint: samples/gpu-pod-cpu-request-missing/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-cpu-request-missing/example_disallowed.yaml + assertions: + - violations: yes +- name: non-gpu-pod + template: template.yaml + constraint: samples/non-gpu-pod/constraint.yaml + cases: + - name: example-allowed + object: samples/non-gpu-pod/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-gpu-request-missing + template: template.yaml + constraint: samples/gpu-pod-gpu-request-missing/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-gpu-request-missing/example_gator_disallowed.yaml + assertions: + - violations: yes +- name: gpu-pod-gpu-mismatch + template: template.yaml + constraint: samples/gpu-pod-gpu-mismatch/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-gpu-mismatch/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-pod-memory-equivalent + template: template.yaml + constraint: samples/gpu-pod-memory-equivalent/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-memory-equivalent/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-exempt + template: template.yaml + constraint: samples/gpu-pod-exempt/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-exempt/example_allowed.yaml + assertions: + - violations: no \ No newline at end of file diff --git a/artifacthub/library/general/gpuworkloadresources/1.0.0/template.yaml b/artifacthub/library/general/gpuworkloadresources/1.0.0/template.yaml new file mode 100644 index 000000000..36c44d29b --- /dev/null +++ b/artifacthub/library/general/gpuworkloadresources/1.0.0/template.yaml @@ -0,0 +1,341 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuworkloadresources + annotations: + metadata.gatekeeper.sh/title: "GPU Workload Resources" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set + memory requests equal to limits for all non-exempt containers and to set + a CPU request. Containers that request GPUs must also set matching GPU + requests and limits. This keeps GPU workloads schedulable and predictable + while still allowing CPU burst above request. +spec: + crd: + spec: + names: + kind: K8sGpuWorkloadResources + validation: + openAPIV3Schema: + type: object + description: >- + Enforces memory request/limit alignment and cpu requests for GPU pods, + plus matching gpu requests and limits on GPU containers. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: enforcedContainers + expression: 'variables.containers + variables.initContainers' + - name: allContainers + expression: 'variables.enforcedContainers + variables.ephemeralContainers' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.allContainers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: gpuContainers + expression: | + variables.allContainers.filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + ( + (has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) || + (has(container.resources.requests) && + "nvidia.com/gpu" in container.resources.requests && + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) + ) + ) + - name: podRequestsGpu + expression: 'size(variables.gpuContainers) > 0' + - name: gpuRequestLimitMessages + expression: | + variables.gpuContainers.filter(container, + !has(container.resources.requests) || + !("nvidia.com/gpu" in container.resources.requests) || + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) <= 0 || + !has(container.resources.limits) || + !("nvidia.com/gpu" in container.resources.limits) || + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) <= 0 || + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity(string(container.resources.limits["nvidia.com/gpu"]))) != 0 + ).map(container, "Container <" + container.name + "> must set nvidia.com/gpu request equal to limit") + - name: memoryRequestLimitMessages + expression: | + !variables.podRequestsGpu ? [] : + variables.enforcedContainers.filter(container, + !(container.image in variables.exemptImages) && + ( + !has(container.resources) || + !has(container.resources.requests) || + !("memory" in container.resources.requests) || + !has(container.resources.limits) || + !("memory" in container.resources.limits) || + quantity(string(container.resources.requests["memory"])).compareTo(quantity(string(container.resources.limits["memory"]))) != 0 + ) + ).map(container, "Container <" + container.name + "> in a GPU pod must set memory request equal to limit") + - name: cpuRequestMessages + expression: | + !variables.podRequestsGpu ? [] : + variables.enforcedContainers.filter(container, + !(container.image in variables.exemptImages) && + ( + !has(container.resources) || + !has(container.resources.requests) || + !("cpu" in container.resources.requests) || + string(container.resources.requests["cpu"]) == "" + ) + ).map(container, "Container <" + container.name + "> in a GPU pod must set a cpu request") + validations: + - expression: 'size(variables.gpuRequestLimitMessages) == 0' + messageExpression: 'variables.gpuRequestLimitMessages.join(", ")' + - expression: 'size(variables.memoryRequestLimitMessages) == 0' + messageExpression: 'variables.memoryRequestLimitMessages.join(", ")' + - expression: 'size(variables.cpuRequestMessages) == 0' + messageExpression: 'variables.cpuRequestMessages.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpuworkloadresources + + import data.lib.exempt_container.is_exempt + + missing(obj, field) = true { + not obj[field] + } + + missing(obj, field) = true { + obj[field] == "" + } + + # 10 ** 21 + mem_multiple("E") = 1000000000000000000000 { true } + + # 10 ** 18 + mem_multiple("P") = 1000000000000000000 { true } + + # 10 ** 15 + mem_multiple("T") = 1000000000000000 { true } + + # 10 ** 12 + mem_multiple("G") = 1000000000000 { true } + + # 10 ** 9 + mem_multiple("M") = 1000000000 { true } + + # 10 ** 6 + mem_multiple("k") = 1000000 { true } + + # 10 ** 3 + mem_multiple("") = 1000 { true } + + # Kubernetes accepts millibyte precision when it probably shouldn't. + # https://github.com/kubernetes/kubernetes/issues/28741 + # 10 ** 0 + mem_multiple("m") = 1 { true } + + # 1000 * 2 ** 10 + mem_multiple("Ki") = 1024000 { true } + + # 1000 * 2 ** 20 + mem_multiple("Mi") = 1048576000 { true } + + # 1000 * 2 ** 30 + mem_multiple("Gi") = 1073741824000 { true } + + # 1000 * 2 ** 40 + mem_multiple("Ti") = 1099511627776000 { true } + + # 1000 * 2 ** 50 + mem_multiple("Pi") = 1125899906842624000 { true } + + # 1000 * 2 ** 60 + mem_multiple("Ei") = 1152921504606846976000 { true } + + get_suffix(mem) = suffix { + not is_string(mem) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 0 + suffix := substring(mem, count(mem) - 1, -1) + mem_multiple(suffix) + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 1 + suffix := substring(mem, count(mem) - 2, -1) + mem_multiple(suffix) + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 1 + not mem_multiple(substring(mem, count(mem) - 1, -1)) + not mem_multiple(substring(mem, count(mem) - 2, -1)) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) == 1 + not mem_multiple(substring(mem, count(mem) - 1, -1)) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) == 0 + suffix := "" + } + + canonify_mem(orig) = new { + is_number(orig) + new := orig * 1000 + } + + canonify_mem(orig) = new { + not is_number(orig) + suffix := get_suffix(orig) + raw := replace(orig, suffix, "") + regex.match("^[0-9]+(\\.[0-9]+)?$", raw) + new := to_number(raw) * mem_multiple(suffix) + } + + violation[{"msg": msg}] { + container := gpu_containers[_] + not has_matching_gpu_request_and_limit(container) + msg := sprintf("Container <%v> must set nvidia.com/gpu request equal to limit", [container.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + container := enforced_containers[_] + not is_exempt(container) + not has_matching_memory_request_and_limit(container) + msg := sprintf("Container <%v> in a GPU pod must set memory request equal to limit", [container.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + container := enforced_containers[_] + not is_exempt(container) + not has_cpu_request(container) + msg := sprintf("Container <%v> in a GPU pod must set a cpu request", [container.name]) + } + + pod_requests_gpu { + gpu_containers[_] + } + + gpu_containers[c] { + c := all_containers[_] + not is_exempt(c) + requests_gpu(c) + } + + enforced_containers[c] { + c := input.review.object.spec.containers[_] + } + + enforced_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + all_containers[c] { + c := enforced_containers[_] + } + + all_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + + requests_gpu(container) { + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu := limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + requests_gpu(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + gpu := requests["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_matching_gpu_request_and_limit(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu_request := requests["nvidia.com/gpu"] + gpu_limit := limits["nvidia.com/gpu"] + to_number(gpu_request) > 0 + to_number(gpu_limit) > 0 + to_number(gpu_request) == to_number(gpu_limit) + } + + has_matching_memory_request_and_limit(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + limits := object.get(object.get(container, "resources", {}), "limits", {}) + mem_request := requests["memory"] + mem_limit := limits["memory"] + canonify_mem(mem_request) == canonify_mem(mem_limit) + } + + has_cpu_request(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + not missing(requests, "cpu") + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } \ No newline at end of file diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/artifacthub-pkg.yml b/artifacthub/library/general/nounsupportedgpu/1.0.0/artifacthub-pkg.yml new file mode 100644 index 000000000..2c9fdd216 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/artifacthub-pkg.yml @@ -0,0 +1,22 @@ +version: 1.0.0 +name: k8snounsupportedgpu +displayName: No Unsupported GPU Requests +createdAt: "2026-03-17T00:04:28Z" +description: Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set the NVIDIA_VISIBLE_DEVICES environment variable, indicating the container image is built to consume GPUs via the NVIDIA CUDA runtime. This prevents GPU resource waste from containers that request GPUs but cannot use them. +digest: a128ee4cb1ec36fada48fc2b85ad617f1e83ae21574a7c2909ce92bb0c9d2ec8 +license: Apache-2.0 +homeURL: https://open-policy-agent.github.io/gatekeeper-library/website/nounsupportedgpu +keywords: + - gatekeeper + - open-policy-agent + - policies +readme: |- + # No Unsupported GPU Requests + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set the NVIDIA_VISIBLE_DEVICES environment variable, indicating the container image is built to consume GPUs via the NVIDIA CUDA runtime. This prevents GPU resource waste from containers that request GPUs but cannot use them. +install: |- + ### Usage + ```shell + kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/artifacthub/library/general/nounsupportedgpu/1.0.0/template.yaml + ``` +provider: + name: Gatekeeper Library diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/kustomization.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-ephemeral-without-env-var/constraint.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-ephemeral-without-env-var/constraint.yaml new file mode 100644 index 000000000..1bf71de18 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-ephemeral-without-env-var/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-ephemeral-without-env-var/example_disallowed.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-ephemeral-without-env-var/example_disallowed.yaml new file mode 100644 index 000000000..1416a5210 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-ephemeral-without-env-var/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-ephemeral-without-env-var +spec: + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" + ephemeralContainers: + - name: debug-gpu + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-exact-exempt/constraint.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-exact-exempt/constraint.yaml new file mode 100644 index 000000000..ea1935cd4 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-exact-exempt/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:3.1.7" \ No newline at end of file diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-exact-exempt/example_allowed.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-exact-exempt/example_allowed.yaml new file mode 100644 index 000000000..775dcb40a --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-exact-exempt/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exact-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-init-without-env-var/constraint.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-init-without-env-var/constraint.yaml new file mode 100644 index 000000000..1bf71de18 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-init-without-env-var/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-init-without-env-var/example_disallowed.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-init-without-env-var/example_disallowed.yaml new file mode 100644 index 000000000..69149470a --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-init-without-env-var/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-init-without-env-var +spec: + initContainers: + - name: setup + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" \ No newline at end of file diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-with-env-var/constraint.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-with-env-var/constraint.yaml new file mode 100644 index 000000000..1fb10c208 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-with-env-var/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-with-env-var/example_allowed.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-with-env-var/example_allowed.yaml new file mode 100644 index 000000000..23e1bc9fd --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-with-env-var/example_allowed.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-allowed +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + env: + - name: NVIDIA_VISIBLE_DEVICES + value: all diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/constraint.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/constraint.yaml new file mode 100644 index 000000000..9e3d493ea --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/example_allowed_exempt.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/example_allowed_exempt.yaml new file mode 100644 index 000000000..56b5d2c1a --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/example_allowed_exempt.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/example_disallowed.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/example_disallowed.yaml new file mode 100644 index 000000000..f276fe228 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/gpu-without-env-var/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-disallowed +spec: + containers: + - name: training + image: myrepo/custom-ml-image:v1 + resources: + limits: + nvidia.com/gpu: "1" diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/no-gpu-requested/constraint.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/no-gpu-requested/constraint.yaml new file mode 100644 index 000000000..1fb10c208 --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/no-gpu-requested/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/no-gpu-requested/example_allowed.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/no-gpu-requested/example_allowed.yaml new file mode 100644 index 000000000..2fc1db2ad --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/samples/no-gpu-requested/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/suite.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/suite.yaml new file mode 100644 index 000000000..1b8b225ee --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/suite.yaml @@ -0,0 +1,57 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: nounsupportedgpu +tests: +- name: gpu-with-env-var + template: template.yaml + constraint: samples/gpu-with-env-var/constraint.yaml + cases: + - name: example-allowed-gpu-with-env + object: samples/gpu-with-env-var/example_allowed.yaml + assertions: + - violations: no +- name: gpu-without-env-var + template: template.yaml + constraint: samples/gpu-without-env-var/constraint.yaml + cases: + - name: example-disallowed-gpu-no-env + object: samples/gpu-without-env-var/example_disallowed.yaml + assertions: + - violations: yes + - name: example-allowed-exempt-image + object: samples/gpu-without-env-var/example_allowed_exempt.yaml + assertions: + - violations: no +- name: no-gpu-requested + template: template.yaml + constraint: samples/no-gpu-requested/constraint.yaml + cases: + - name: example-allowed-no-gpu + object: samples/no-gpu-requested/example_allowed.yaml + assertions: + - violations: no +- name: gpu-init-without-env-var + template: template.yaml + constraint: samples/gpu-init-without-env-var/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-init-without-env-var/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-ephemeral-without-env-var + template: template.yaml + constraint: samples/gpu-ephemeral-without-env-var/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-ephemeral-without-env-var/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-exact-exempt + template: template.yaml + constraint: samples/gpu-exact-exempt/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exact-exempt/example_allowed.yaml + assertions: + - violations: no diff --git a/artifacthub/library/general/nounsupportedgpu/1.0.0/template.yaml b/artifacthub/library/general/nounsupportedgpu/1.0.0/template.yaml new file mode 100644 index 000000000..bd6ce987d --- /dev/null +++ b/artifacthub/library/general/nounsupportedgpu/1.0.0/template.yaml @@ -0,0 +1,130 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8snounsupportedgpu + annotations: + metadata.gatekeeper.sh/title: "No Unsupported GPU Requests" + metadata.gatekeeper.sh/version: 1.0.0 + description: >- + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set + the NVIDIA_VISIBLE_DEVICES environment variable, indicating the container + image is built to consume GPUs via the NVIDIA CUDA runtime. This prevents + GPU resource waste from containers that request GPUs but cannot use them. +spec: + crd: + spec: + names: + kind: K8sNoUnsupportedGpu + validation: + openAPIV3Schema: + type: object + description: >- + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set + the NVIDIA_VISIBLE_DEVICES environment variable. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: badContainers + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + (!has(container.env) || !container.env.exists(e, e.name == "NVIDIA_VISIBLE_DEVICES")) + ).map(container, "Container <" + container.name + "> requests nvidia.com/gpu but does not set the NVIDIA_VISIBLE_DEVICES environment variable") + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8snounsupportedgpu + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input_containers[_] + not is_exempt(container) + has_gpu_request(container) + not has_nvidia_env(container) + msg := sprintf("Container <%v> requests nvidia.com/gpu but does not set the NVIDIA_VISIBLE_DEVICES environment variable", [container.name]) + } + + has_gpu_request(container) { + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_nvidia_env(container) { + env := container.env[_] + env.name == "NVIDIA_VISIBLE_DEVICES" + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/artifacthub-pkg.yml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/artifacthub-pkg.yml new file mode 100644 index 000000000..fc93fdcf2 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/artifacthub-pkg.yml @@ -0,0 +1,22 @@ +version: 1.0.0 +name: k8srequiredgpuruntimeclass +displayName: Required GPU Runtime Class +createdAt: "2026-03-17T00:04:29Z" +description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to specify a runtimeClassName from an allowed list. This ensures GPU workloads use the proper container runtime (e.g., nvidia) rather than relying on default runtime hooks. +digest: 5197dff6a2680b6eccb1a805f75df87a2772d702867e25ad8be509166f19d41b +license: Apache-2.0 +homeURL: https://open-policy-agent.github.io/gatekeeper-library/website/requiredgpuruntimeclass +keywords: + - gatekeeper + - open-policy-agent + - policies +readme: |- + # Required GPU Runtime Class + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to specify a runtimeClassName from an allowed list. This ensures GPU workloads use the proper container runtime (e.g., nvidia) rather than relying on default runtime hooks. +install: |- + ### Usage + ```shell + kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/template.yaml + ``` +provider: + name: Gatekeeper Library diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/kustomization.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-exempt-without-runtimeclass/constraint.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-exempt-without-runtimeclass/constraint.yaml new file mode 100644 index 000000000..bc8bcf719 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-exempt-without-runtimeclass/constraint.yaml @@ -0,0 +1,14 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-exempt-without-runtimeclass/example_allowed.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-exempt-without-runtimeclass/example_allowed.yaml new file mode 100644 index 000000000..03c95f066 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-exempt-without-runtimeclass/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-runtimeclass +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-runtimeclass-disabled/constraint.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-runtimeclass-disabled/constraint.yaml new file mode 100644 index 000000000..b45ecf2a0 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-runtimeclass-disabled/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-runtimeclass-disabled/example_allowed.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-runtimeclass-disabled/example_allowed.yaml new file mode 100644 index 000000000..83d99b8fd --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-runtimeclass-disabled/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-runtimeclass-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-with-runtimeclass/constraint.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-with-runtimeclass/constraint.yaml new file mode 100644 index 000000000..2ba97b990 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-with-runtimeclass/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-with-runtimeclass/example_allowed.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-with-runtimeclass/example_allowed.yaml new file mode 100644 index 000000000..0f0111cd0 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-with-runtimeclass/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-runtime +spec: + runtimeClassName: nvidia + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-without-runtimeclass/constraint.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-without-runtimeclass/constraint.yaml new file mode 100644 index 000000000..2ba97b990 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-without-runtimeclass/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-without-runtimeclass/example_disallowed.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-without-runtimeclass/example_disallowed.yaml new file mode 100644 index 000000000..e98db8edd --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-without-runtimeclass/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-runtime +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-wrong-runtimeclass/constraint.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-wrong-runtimeclass/constraint.yaml new file mode 100644 index 000000000..fe1939500 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-wrong-runtimeclass/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-wrong-runtimeclass/example_disallowed.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-wrong-runtimeclass/example_disallowed.yaml new file mode 100644 index 000000000..d40249d32 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/gpu-wrong-runtimeclass/example_disallowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-wrong-runtimeclass +spec: + runtimeClassName: runc + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/no-gpu/constraint.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/no-gpu/constraint.yaml new file mode 100644 index 000000000..2ba97b990 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/no-gpu/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/no-gpu/example_allowed.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/no-gpu/example_allowed.yaml new file mode 100644 index 000000000..2fc1db2ad --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/samples/no-gpu/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/suite.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/suite.yaml new file mode 100644 index 000000000..9eec23598 --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/suite.yaml @@ -0,0 +1,53 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: requiredgpuruntimeclass +tests: +- name: gpu-with-runtimeclass + template: template.yaml + constraint: samples/gpu-with-runtimeclass/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-with-runtimeclass/example_allowed.yaml + assertions: + - violations: no +- name: gpu-without-runtimeclass + template: template.yaml + constraint: samples/gpu-without-runtimeclass/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-without-runtimeclass/example_disallowed.yaml + assertions: + - violations: yes +- name: no-gpu + template: template.yaml + constraint: samples/no-gpu/constraint.yaml + cases: + - name: example-allowed + object: samples/no-gpu/example_allowed.yaml + assertions: + - violations: no +- name: gpu-wrong-runtimeclass + template: template.yaml + constraint: samples/gpu-wrong-runtimeclass/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-wrong-runtimeclass/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-runtimeclass-disabled + template: template.yaml + constraint: samples/gpu-runtimeclass-disabled/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-runtimeclass-disabled/example_allowed.yaml + assertions: + - violations: no +- name: gpu-exempt-without-runtimeclass + template: template.yaml + constraint: samples/gpu-exempt-without-runtimeclass/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exempt-without-runtimeclass/example_allowed.yaml + assertions: + - violations: no diff --git a/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/template.yaml b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/template.yaml new file mode 100644 index 000000000..452ba13eb --- /dev/null +++ b/artifacthub/library/general/requiredgpuruntimeclass/1.0.0/template.yaml @@ -0,0 +1,139 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8srequiredgpuruntimeclass + annotations: + metadata.gatekeeper.sh/title: "Required GPU Runtime Class" + metadata.gatekeeper.sh/version: 1.0.0 + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to specify + a runtimeClassName from an allowed list. This ensures GPU workloads use + the proper container runtime (e.g., nvidia) rather than relying on default + runtime hooks. +spec: + crd: + spec: + names: + kind: K8sRequiredGpuRuntimeClass + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to specify an allowed runtimeClassName. + properties: + allowedRuntimeClassNames: + description: >- + List of allowed runtime class names for GPU workloads (e.g., ["nvidia"]). + type: array + items: + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: allowedRuntimeClassNames + expression: 'has(variables.params.allowedRuntimeClassNames) ? variables.params.allowedRuntimeClassNames : []' + - name: hasAllowedRc + expression: | + !has(variables.anyObject.spec.runtimeClassName) ? false : + variables.anyObject.spec.runtimeClassName in variables.allowedRuntimeClassNames + validations: + - expression: '!variables.podRequestsGpu || size(variables.allowedRuntimeClassNames) == 0 || variables.hasAllowedRc' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not specify an allowed runtimeClassName (allowed: " + variables.allowedRuntimeClassNames.join(", ") + ")"' + - engine: Rego + source: + rego: | + package k8srequiredgpuruntimeclass + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + allowed := object.get(input, ["parameters", "allowedRuntimeClassNames"], []) + count(allowed) > 0 + not has_allowed_runtime_class(allowed) + msg := sprintf("Pod <%v> requests GPU resources but does not specify an allowed runtimeClassName (allowed: %v)", [input.review.object.metadata.name, allowed]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_allowed_runtime_class(allowed) { + rc := input.review.object.spec.runtimeClassName + rc == allowed[_] + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/artifacthub-pkg.yml b/artifacthub/library/general/requiredgputoleration/1.0.0/artifacthub-pkg.yml new file mode 100644 index 000000000..4b37db82a --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/artifacthub-pkg.yml @@ -0,0 +1,22 @@ +version: 1.0.0 +name: k8srequiredgputoleration +displayName: Required GPU Toleration +createdAt: "2026-03-17T00:04:29Z" +description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to include a toleration for the specified GPU node taint. This ensures GPU workloads are properly configured to schedule on GPU nodes. +digest: fa8687581fa25bc9a772f086440b04f8fcc594dd5eed0215dad02563255eee3c +license: Apache-2.0 +homeURL: https://open-policy-agent.github.io/gatekeeper-library/website/requiredgputoleration +keywords: + - gatekeeper + - open-policy-agent + - policies +readme: |- + # Required GPU Toleration + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to include a toleration for the specified GPU node taint. This ensures GPU workloads are properly configured to schedule on GPU nodes. +install: |- + ### Usage + ```shell + kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/artifacthub/library/general/requiredgputoleration/1.0.0/template.yaml + ``` +provider: + name: Gatekeeper Library diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/kustomization.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-exempt-without-toleration/constraint.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-exempt-without-toleration/constraint.yaml new file mode 100644 index 000000000..153d430e7 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-exempt-without-toleration/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-exempt-without-toleration/example_allowed.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-exempt-without-toleration/example_allowed.yaml new file mode 100644 index 000000000..71d46d9f0 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-exempt-without-toleration/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-toleration +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-disabled/constraint.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-disabled/constraint.yaml new file mode 100644 index 000000000..6e3a75317 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-disabled/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-disabled/example_allowed.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-disabled/example_allowed.yaml new file mode 100644 index 000000000..485ae9a4b --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-disabled/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-toleration-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-key-only/constraint.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-key-only/constraint.yaml new file mode 100644 index 000000000..9bce00a68 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-key-only/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-key-only/example_allowed.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-key-only/example_allowed.yaml new file mode 100644 index 000000000..1a992fcaa --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-toleration-key-only/example_allowed.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-toleration-key-only +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "present" + effect: "NoExecute" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-with-toleration/constraint.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-with-toleration/constraint.yaml new file mode 100644 index 000000000..ba4ce180c --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-with-toleration/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-with-toleration/example_allowed.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-with-toleration/example_allowed.yaml new file mode 100644 index 000000000..46a3d0af3 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-with-toleration/example_allowed.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-without-toleration/constraint.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-without-toleration/constraint.yaml new file mode 100644 index 000000000..ba4ce180c --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-without-toleration/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-without-toleration/example_disallowed.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-without-toleration/example_disallowed.yaml new file mode 100644 index 000000000..9cd325882 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-without-toleration/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-wrong-toleration/constraint.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-wrong-toleration/constraint.yaml new file mode 100644 index 000000000..9bce00a68 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-wrong-toleration/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-wrong-toleration/example_disallowed.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-wrong-toleration/example_disallowed.yaml new file mode 100644 index 000000000..3d866fa43 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/gpu-wrong-toleration/example_disallowed.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-wrong-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "workload" + operator: "Exists" + effect: "NoSchedule" \ No newline at end of file diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/no-gpu/constraint.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/no-gpu/constraint.yaml new file mode 100644 index 000000000..ba4ce180c --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/no-gpu/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/samples/no-gpu/example_allowed.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/no-gpu/example_allowed.yaml new file mode 100644 index 000000000..2fc1db2ad --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/samples/no-gpu/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/suite.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/suite.yaml new file mode 100644 index 000000000..d53dd3af7 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/suite.yaml @@ -0,0 +1,61 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: requiredgputoleration +tests: +- name: gpu-with-toleration + template: template.yaml + constraint: samples/gpu-with-toleration/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-with-toleration/example_allowed.yaml + assertions: + - violations: no +- name: gpu-without-toleration + template: template.yaml + constraint: samples/gpu-without-toleration/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-without-toleration/example_disallowed.yaml + assertions: + - violations: yes +- name: no-gpu + template: template.yaml + constraint: samples/no-gpu/constraint.yaml + cases: + - name: example-allowed + object: samples/no-gpu/example_allowed.yaml + assertions: + - violations: no +- name: gpu-wrong-toleration + template: template.yaml + constraint: samples/gpu-wrong-toleration/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-wrong-toleration/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-toleration-disabled + template: template.yaml + constraint: samples/gpu-toleration-disabled/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-toleration-disabled/example_allowed.yaml + assertions: + - violations: no +- name: gpu-exempt-without-toleration + template: template.yaml + constraint: samples/gpu-exempt-without-toleration/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exempt-without-toleration/example_allowed.yaml + assertions: + - violations: no +- name: gpu-toleration-key-only + template: template.yaml + constraint: samples/gpu-toleration-key-only/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-toleration-key-only/example_allowed.yaml + assertions: + - violations: no diff --git a/artifacthub/library/general/requiredgputoleration/1.0.0/template.yaml b/artifacthub/library/general/requiredgputoleration/1.0.0/template.yaml new file mode 100644 index 000000000..01baef028 --- /dev/null +++ b/artifacthub/library/general/requiredgputoleration/1.0.0/template.yaml @@ -0,0 +1,137 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8srequiredgputoleration + annotations: + metadata.gatekeeper.sh/title: "Required GPU Toleration" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to include + a toleration for the specified GPU node taint. This ensures GPU workloads + are properly configured to schedule on GPU nodes. +spec: + crd: + spec: + names: + kind: K8sRequiredGpuToleration + validation: + openAPIV3Schema: + type: object + description: >- + Requires pods requesting GPUs to include a specified toleration. + properties: + tolerationKey: + description: >- + The taint key that GPU pods must tolerate (e.g., "nvidia.com/gpu"). + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: tolerationKey + expression: 'has(variables.params.tolerationKey) ? variables.params.tolerationKey : ""' + - name: hasToleration + expression: | + !has(variables.anyObject.spec.tolerations) ? false : + variables.anyObject.spec.tolerations.exists(t, t.key == variables.tolerationKey) + validations: + - expression: '!variables.podRequestsGpu || variables.tolerationKey == "" || variables.hasToleration' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not tolerate taint key <" + variables.tolerationKey + ">"' + - engine: Rego + source: + rego: | + package k8srequiredgputoleration + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + toleration_key := object.get(input, ["parameters", "tolerationKey"], "") + toleration_key != "" + not has_toleration(toleration_key) + msg := sprintf("Pod <%v> requests GPU resources but does not tolerate taint key <%v>", [input.review.object.metadata.name, toleration_key]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_toleration(key) { + toleration := input.review.object.spec.tolerations[_] + toleration.key == key + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/catalog.yaml b/catalog.yaml index 73a77e51d..0ba3f84f4 100644 --- a/catalog.yaml +++ b/catalog.yaml @@ -1,13 +1,42 @@ # Policy Catalog for Gatekeeper Library # # This file is auto-generated. Do not edit directly. -# Generated at: 2026-02-12T23:49:33Z +# Generated at: 2026-04-17T16:47:03Z # # To regenerate, run: # gator policy generate-catalog --library-path=. --output=catalog.yaml # apiVersion: gator.gatekeeper.sh/v1alpha1 bundles: +- description: Enforces policies commonly required for AI inference workloads. Includes + GPU safety controls for predictable scheduling and resource configuration without + training-specific requirements. + name: gatekeeper-ai-inference-policies + policies: + - k8sgpunodetargeting + - k8sgpuresourcelimits + - k8sgpuworkloadresources + - k8srequiredgputoleration +- description: Enforces policies commonly required for AI training workloads. Includes + GPU safety controls plus training-specific safeguards such as active deadlines + and shared memory configuration. + name: gatekeeper-ai-training-policies + policies: + - k8sgpuactivedeadline + - k8sgpunodetargeting + - k8sgpuresourcelimits + - k8sgpusharedmemory + - k8sgpuworkloadresources + - k8srequiredgputoleration +- description: Enforces baseline safety controls for GPU workloads. Prevents GPU resource + waste, ensures proper scheduling configuration, and requires predictable GPU resource + requests. + name: gatekeeper-gpu-safety-policies + policies: + - k8sgpunodetargeting + - k8sgpuresourcelimits + - k8sgpuworkloadresources + - k8srequiredgputoleration - description: |- Enforces Pod Security Standards at Baseline level. Prevents known privilege escalations. See https://kubernetes.io/docs/concepts/security/pod-security-standards/ @@ -49,7 +78,7 @@ kind: PolicyCatalog metadata: name: gatekeeper-library repository: https://github.com/open-policy-agent/gatekeeper-library - updatedAt: "2026-02-12T23:49:33.304069431Z" + updatedAt: "2026-04-17T16:47:03.417872793Z" version: v1.0.0 policies: - category: general @@ -161,6 +190,76 @@ policies: name: k8sexternalips templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/externalip/template.yaml version: v1.0.0 +- bundleConstraints: + gatekeeper-ai-training-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/constraint.yaml + bundles: + - gatekeeper-ai-training-policies + category: general + description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to + set activeDeadlineSeconds. This prevents runaway training jobs from holding GPU + resources indefinitely. + name: k8sgpuactivedeadline + templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuactivedeadline/template.yaml + version: v1.0.0 +- bundleConstraints: + gatekeeper-ai-inference-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/constraint.yaml + gatekeeper-ai-training-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/constraint.yaml + gatekeeper-gpu-safety-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/constraint.yaml + bundles: + - gatekeeper-gpu-safety-policies + - gatekeeper-ai-training-policies + - gatekeeper-ai-inference-policies + category: general + description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to + target GPU-labeled nodes using required node affinity or nodeSelector. This helps + ensure GPU workloads only land on nodes that advertise GPU capacity. + name: k8sgpunodetargeting + templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpunodetargeting/template.yaml + version: v1.0.0 +- bundleConstraints: + gatekeeper-ai-inference-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/constraint.yaml + gatekeeper-ai-training-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/constraint.yaml + gatekeeper-gpu-safety-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/constraint.yaml + bundles: + - gatekeeper-gpu-safety-policies + - gatekeeper-ai-training-policies + - gatekeeper-ai-inference-policies + category: general + description: Enforces a maximum number of NVIDIA GPUs (nvidia.com/gpu) that a single + container may request. This prevents individual containers from hoarding GPU resources + on shared clusters, particularly for AI/ML training workloads. + name: k8sgpuresourcelimits + templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuresourcelimits/template.yaml + version: v1.0.0 +- bundleConstraints: + gatekeeper-ai-training-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpusharedmemory/samples/gpu-with-shm/constraint.yaml + bundles: + - gatekeeper-ai-training-policies + category: general + description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to + mount a memory-backed emptyDir volume at /dev/shm. PyTorch DataLoader, NCCL multi-GPU + communication, and most training frameworks require shared memory beyond the default + 64MB. + name: k8sgpusharedmemory + templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpusharedmemory/template.yaml + version: v1.0.0 +- bundleConstraints: + gatekeeper-ai-inference-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuworkloadresources/samples/gpu-pod-compliant/constraint.yaml + gatekeeper-ai-training-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuworkloadresources/samples/gpu-pod-compliant/constraint.yaml + gatekeeper-gpu-safety-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuworkloadresources/samples/gpu-pod-compliant/constraint.yaml + bundles: + - gatekeeper-gpu-safety-policies + - gatekeeper-ai-training-policies + - gatekeeper-ai-inference-policies + category: general + description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to + set memory requests equal to limits for all non-exempt containers and to set a + CPU request. Containers that request GPUs must also set matching GPU requests + and limits. This keeps GPU workloads schedulable and predictable while still allowing + CPU burst above request. + name: k8sgpuworkloadresources + templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/gpuworkloadresources/template.yaml + version: v1.0.0 - category: general description: Disallow the following scenarios when deploying `HorizontalPodAutoscalers` 1. Deployment of HorizontalPodAutoscalers with `.spec.minReplicas` or `.spec.maxReplicas` @@ -186,6 +285,14 @@ policies: name: k8simagedigests templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/imagedigests/template.yaml version: v1.0.1 +- category: general + description: Containers which request NVIDIA GPU resources (nvidia.com/gpu) must + set the NVIDIA_VISIBLE_DEVICES environment variable, indicating the container + image is built to consume GPUs via the NVIDIA CUDA runtime. This prevents GPU + resource waste from containers that request GPUs but cannot use them. + name: k8snounsupportedgpu + templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/nounsupportedgpu/template.yaml + version: v1.0.0 - category: general description: |- Disallow the following scenarios when deploying PodDisruptionBudgets or resources that implement the replica subresource (e.g. Deployment, ReplicationController, ReplicaSet, StatefulSet): 1. Deployment of PodDisruptionBudgets with .spec.maxUnavailable == 0 2. Deployment of PodDisruptionBudgets with .spec.minAvailable == .spec.replicas of the resource with replica subresource This will prevent PodDisruptionBudgets from blocking voluntary disruptions such as node draining. @@ -429,6 +536,29 @@ policies: name: k8srequiredannotations templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/requiredannotations/template.yaml version: v1.0.1 +- category: general + description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to + specify a runtimeClassName from an allowed list. This ensures GPU workloads use + the proper container runtime (e.g., nvidia) rather than relying on default runtime + hooks. + name: k8srequiredgpuruntimeclass + templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/requiredgpuruntimeclass/template.yaml + version: v1.0.0 +- bundleConstraints: + gatekeeper-ai-inference-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/requiredgputoleration/samples/gpu-with-toleration/constraint.yaml + gatekeeper-ai-training-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/requiredgputoleration/samples/gpu-with-toleration/constraint.yaml + gatekeeper-gpu-safety-policies: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/requiredgputoleration/samples/gpu-with-toleration/constraint.yaml + bundles: + - gatekeeper-gpu-safety-policies + - gatekeeper-ai-training-policies + - gatekeeper-ai-inference-policies + category: general + description: Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to + include a toleration for the specified GPU node taint. This ensures GPU workloads + are properly configured to schedule on GPU nodes. + name: k8srequiredgputoleration + templatePath: https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/refs/heads/master/library/general/requiredgputoleration/template.yaml + version: v1.0.0 - category: general description: Requires resources to contain specified labels, with values matching provided regular expressions. diff --git a/go.work b/go.work index 0866d5e29..baedf8b0b 100755 --- a/go.work +++ b/go.work @@ -1,4 +1,4 @@ -go 1.20 +go 1.23 use ( ./scripts/artifacthub diff --git a/library/general/gpuactivedeadline/kustomization.yaml b/library/general/gpuactivedeadline/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/library/general/gpuactivedeadline/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/library/general/gpuactivedeadline/samples/gpu-job-exempt/constraint.yaml b/library/general/gpuactivedeadline/samples/gpu-job-exempt/constraint.yaml new file mode 100644 index 000000000..43cf7905a --- /dev/null +++ b/library/general/gpuactivedeadline/samples/gpu-job-exempt/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxActiveDeadlineSeconds: 86400 + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/library/general/gpuactivedeadline/samples/gpu-job-exempt/example_allowed.yaml b/library/general/gpuactivedeadline/samples/gpu-job-exempt/example_allowed.yaml new file mode 100644 index 000000000..6b457f9f8 --- /dev/null +++ b/library/general/gpuactivedeadline/samples/gpu-job-exempt/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/constraint.yaml b/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/constraint.yaml new file mode 100644 index 000000000..9b2f2b40a --- /dev/null +++ b/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxActiveDeadlineSeconds: 86400 diff --git a/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/example_allowed.yaml b/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/example_allowed.yaml new file mode 100644 index 000000000..ee9a81aea --- /dev/null +++ b/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-with-deadline +spec: + activeDeadlineSeconds: 3600 + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml b/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml new file mode 100644 index 000000000..eeb36103b --- /dev/null +++ b/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-exceeds-deadline +spec: + activeDeadlineSeconds: 172800 + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpuactivedeadline/samples/gpu-job-without-deadline/constraint.yaml b/library/general/gpuactivedeadline/samples/gpu-job-without-deadline/constraint.yaml new file mode 100644 index 000000000..71df2680f --- /dev/null +++ b/library/general/gpuactivedeadline/samples/gpu-job-without-deadline/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/library/general/gpuactivedeadline/samples/gpu-job-without-deadline/example_disallowed.yaml b/library/general/gpuactivedeadline/samples/gpu-job-without-deadline/example_disallowed.yaml new file mode 100644 index 000000000..11a08f60b --- /dev/null +++ b/library/general/gpuactivedeadline/samples/gpu-job-without-deadline/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-without-deadline +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/library/general/gpuactivedeadline/samples/non-gpu-job/constraint.yaml b/library/general/gpuactivedeadline/samples/non-gpu-job/constraint.yaml new file mode 100644 index 000000000..71df2680f --- /dev/null +++ b/library/general/gpuactivedeadline/samples/non-gpu-job/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/library/general/gpuactivedeadline/samples/non-gpu-job/example_allowed.yaml b/library/general/gpuactivedeadline/samples/non-gpu-job/example_allowed.yaml new file mode 100644 index 000000000..0c7634b44 --- /dev/null +++ b/library/general/gpuactivedeadline/samples/non-gpu-job/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-job +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/library/general/gpuactivedeadline/suite.yaml b/library/general/gpuactivedeadline/suite.yaml new file mode 100644 index 000000000..d9c338124 --- /dev/null +++ b/library/general/gpuactivedeadline/suite.yaml @@ -0,0 +1,41 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpuactivedeadline +tests: +- name: gpu-job-with-deadline + template: template.yaml + constraint: samples/gpu-job-with-deadline/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-job-with-deadline/example_allowed.yaml + assertions: + - violations: no + - name: example-disallowed-exceeds-max + object: samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml + assertions: + - violations: yes +- name: gpu-job-without-deadline + template: template.yaml + constraint: samples/gpu-job-without-deadline/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-job-without-deadline/example_disallowed.yaml + assertions: + - violations: yes +- name: non-gpu-job + template: template.yaml + constraint: samples/non-gpu-job/constraint.yaml + cases: + - name: example-allowed + object: samples/non-gpu-job/example_allowed.yaml + assertions: + - violations: no +- name: gpu-job-exempt + template: template.yaml + constraint: samples/gpu-job-exempt/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-job-exempt/example_allowed.yaml + assertions: + - violations: no diff --git a/library/general/gpuactivedeadline/template.yaml b/library/general/gpuactivedeadline/template.yaml new file mode 100644 index 000000000..677d2134b --- /dev/null +++ b/library/general/gpuactivedeadline/template.yaml @@ -0,0 +1,145 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuactivedeadline + annotations: + metadata.gatekeeper.sh/title: "GPU Active Deadline Required" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-ai-training-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set + activeDeadlineSeconds. This prevents runaway training jobs from holding + GPU resources indefinitely. +spec: + crd: + spec: + names: + kind: K8sGpuActiveDeadline + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to set activeDeadlineSeconds. + properties: + maxActiveDeadlineSeconds: + description: >- + The maximum value allowed for activeDeadlineSeconds. Set to 0 to + only require the field is present without enforcing a maximum. + type: integer + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: hasDeadline + expression: 'has(variables.anyObject.spec.activeDeadlineSeconds)' + - name: maxDeadline + expression: 'has(variables.params.maxActiveDeadlineSeconds) ? variables.params.maxActiveDeadlineSeconds : 0' + validations: + - expression: '!variables.podRequestsGpu || variables.hasDeadline' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not set activeDeadlineSeconds"' + - expression: '!variables.podRequestsGpu || !variables.hasDeadline || variables.maxDeadline == 0 || variables.anyObject.spec.activeDeadlineSeconds <= variables.maxDeadline' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> sets activeDeadlineSeconds to " + string(variables.anyObject.spec.activeDeadlineSeconds) + ", which exceeds the maximum allowed " + string(variables.maxDeadline)' + - engine: Rego + source: + rego: | + package k8sgpuactivedeadline + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + not has_active_deadline + msg := sprintf("Pod <%v> requests GPU resources but does not set activeDeadlineSeconds", [input.review.object.metadata.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + has_active_deadline + max_deadline := object.get(input, ["parameters", "maxActiveDeadlineSeconds"], 0) + max_deadline > 0 + deadline := input.review.object.spec.activeDeadlineSeconds + deadline > max_deadline + msg := sprintf("Pod <%v> sets activeDeadlineSeconds to %v, which exceeds the maximum allowed %v", [input.review.object.metadata.name, deadline, max_deadline]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_active_deadline { + input.review.object.spec.activeDeadlineSeconds + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/library/general/gpunodetargeting/kustomization.yaml b/library/general/gpunodetargeting/kustomization.yaml new file mode 100644 index 000000000..24dedaea1 --- /dev/null +++ b/library/general/gpunodetargeting/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-exempt/constraint.yaml b/library/general/gpunodetargeting/samples/gpu-pod-exempt/constraint.yaml new file mode 100644 index 000000000..37f2a0d77 --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-exempt/constraint.yaml @@ -0,0 +1,15 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-exempt/example_allowed.yaml b/library/general/gpunodetargeting/samples/gpu-pod-exempt/example_allowed.yaml new file mode 100644 index 000000000..9f8e7c5af --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-exempt/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/constraint.yaml b/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/constraint.yaml new file mode 100644 index 000000000..5ee41f321 --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/example_allowed.yaml b/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/example_allowed.yaml new file mode 100644 index 000000000..001651b65 --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/example_allowed.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-node-selector-key-only +spec: + nodeSelector: + nvidia.com/gpu.present: "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml b/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml new file mode 100644 index 000000000..dfbbf24b9 --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-node-selector-empty-value +spec: + nodeSelector: + nvidia.com/gpu.present: "" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/constraint.yaml b/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/constraint.yaml new file mode 100644 index 000000000..9f299ce0e --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_allowed.yaml b/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_allowed.yaml new file mode 100644 index 000000000..9ecc45dea --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_allowed.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml b/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml new file mode 100644 index 000000000..9b6121ac5 --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml @@ -0,0 +1,21 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-mixed-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + - "false" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml b/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml new file mode 100644 index 000000000..6db6bdee4 --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-wrong-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "false" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-with-node-selector/constraint.yaml b/library/general/gpunodetargeting/samples/gpu-pod-with-node-selector/constraint.yaml new file mode 100644 index 000000000..9f299ce0e --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-with-node-selector/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-with-node-selector/example_allowed.yaml b/library/general/gpunodetargeting/samples/gpu-pod-with-node-selector/example_allowed.yaml new file mode 100644 index 000000000..3c74a2859 --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-with-node-selector/example_allowed.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-node-selector +spec: + nodeSelector: + nvidia.com/gpu.present: "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-without-targeting/constraint.yaml b/library/general/gpunodetargeting/samples/gpu-pod-without-targeting/constraint.yaml new file mode 100644 index 000000000..9f299ce0e --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-without-targeting/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/gpu-pod-without-targeting/example_disallowed.yaml b/library/general/gpunodetargeting/samples/gpu-pod-without-targeting/example_disallowed.yaml new file mode 100644 index 000000000..c02e06e6d --- /dev/null +++ b/library/general/gpunodetargeting/samples/gpu-pod-without-targeting/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-targeting +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/non-gpu-pod/constraint.yaml b/library/general/gpunodetargeting/samples/non-gpu-pod/constraint.yaml new file mode 100644 index 000000000..9f299ce0e --- /dev/null +++ b/library/general/gpunodetargeting/samples/non-gpu-pod/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" \ No newline at end of file diff --git a/library/general/gpunodetargeting/samples/non-gpu-pod/example_allowed.yaml b/library/general/gpunodetargeting/samples/non-gpu-pod/example_allowed.yaml new file mode 100644 index 000000000..492a70fdf --- /dev/null +++ b/library/general/gpunodetargeting/samples/non-gpu-pod/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" \ No newline at end of file diff --git a/library/general/gpunodetargeting/suite.yaml b/library/general/gpunodetargeting/suite.yaml new file mode 100644 index 000000000..69dd8420d --- /dev/null +++ b/library/general/gpunodetargeting/suite.yaml @@ -0,0 +1,65 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpunodetargeting +tests: +- name: gpu-pod-with-node-affinity + template: template.yaml + constraint: samples/gpu-pod-with-node-affinity/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-with-node-affinity/example_allowed.yaml + assertions: + - violations: no + - name: example-disallowed-wrong-value + object: samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml + assertions: + - violations: yes + - name: example-disallowed-mixed-values + object: samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml + assertions: + - violations: yes +- name: gpu-pod-with-node-selector + template: template.yaml + constraint: samples/gpu-pod-with-node-selector/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-with-node-selector/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-without-targeting + template: template.yaml + constraint: samples/gpu-pod-without-targeting/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-without-targeting/example_disallowed.yaml + assertions: + - violations: yes +- name: non-gpu-pod + template: template.yaml + constraint: samples/non-gpu-pod/constraint.yaml + cases: + - name: example-allowed + object: samples/non-gpu-pod/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-node-selector-key-only + template: template.yaml + constraint: samples/gpu-pod-node-selector-key-only/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-node-selector-key-only/example_allowed.yaml + assertions: + - violations: no + - name: example-disallowed-empty-value + object: samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml + assertions: + - violations: yes +- name: gpu-pod-exempt + template: template.yaml + constraint: samples/gpu-pod-exempt/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-exempt/example_allowed.yaml + assertions: + - violations: no \ No newline at end of file diff --git a/library/general/gpunodetargeting/template.yaml b/library/general/gpunodetargeting/template.yaml new file mode 100644 index 000000000..2019d5ea5 --- /dev/null +++ b/library/general/gpunodetargeting/template.yaml @@ -0,0 +1,245 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpunodetargeting + annotations: + metadata.gatekeeper.sh/title: "GPU Node Targeting" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to target + GPU-labeled nodes using required node affinity or nodeSelector. This helps + ensure GPU workloads only land on nodes that advertise GPU capacity. +spec: + crd: + spec: + names: + kind: K8sGpuNodeTargeting + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to target nodes with a configured GPU label key and optional values. + properties: + nodeLabelKey: + description: >- + The node label key that GPU workloads must target (for example, `nvidia.com/gpu.present` + or `nvidia.com/gpu.product`). + type: string + nodeLabelValues: + description: >- + Optional allowed values for the GPU node label. If omitted, the policy only requires the + label key to be present. + type: array + items: + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: allContainers + expression: 'variables.containers + variables.initContainers + variables.ephemeralContainers' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.allContainers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + variables.allContainers.exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + ( + (has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) || + (has(container.resources.requests) && + "nvidia.com/gpu" in container.resources.requests && + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) + ) + ) + - name: nodeLabelKey + expression: 'has(variables.params.nodeLabelKey) ? variables.params.nodeLabelKey : ""' + - name: nodeLabelValues + expression: 'has(variables.params.nodeLabelValues) ? variables.params.nodeLabelValues : []' + - name: hasMatchingNodeSelector + expression: | + !has(variables.anyObject.spec.nodeSelector) || !(variables.nodeLabelKey in variables.anyObject.spec.nodeSelector) ? false : + (size(variables.nodeLabelValues) == 0 ? + string(variables.anyObject.spec.nodeSelector[variables.nodeLabelKey]) != "" : + variables.anyObject.spec.nodeSelector[variables.nodeLabelKey] in variables.nodeLabelValues) + - name: hasMatchingNodeAffinity + expression: | + !has(variables.anyObject.spec.affinity) || + !has(variables.anyObject.spec.affinity.nodeAffinity) || + !has(variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution) || + !has(variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms) ? false : + variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms.exists(term, + has(term.matchExpressions) && + term.matchExpressions.exists(expr, + expr.key == variables.nodeLabelKey && + ( + size(variables.nodeLabelValues) == 0 ? + expr.operator == "Exists" : + expr.operator == "In" && + has(expr.values) && + size(expr.values) > 0 && + expr.values.all(exprValue, exprValue in variables.nodeLabelValues) + ) + ) + ) + validations: + - expression: '!variables.podRequestsGpu || variables.nodeLabelKey == "" || variables.hasMatchingNodeSelector || variables.hasMatchingNodeAffinity' + messageExpression: | + size(variables.nodeLabelValues) == 0 ? + "Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not target nodes with label key <" + variables.nodeLabelKey + "> using node affinity or nodeSelector" : + "Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not target nodes with label <" + variables.nodeLabelKey + "> matching one of <" + variables.nodeLabelValues.join(", ") + "> using node affinity or nodeSelector" + - engine: Rego + source: + rego: | + package k8sgpunodetargeting + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + label_key := object.get(input.parameters, "nodeLabelKey", "") + label_key != "" + not has_matching_node_selector(label_key) + not has_matching_node_affinity(label_key) + label_values := object.get(input.parameters, "nodeLabelValues", []) + msg := violation_message(label_key, label_values) + } + + violation_message(label_key, label_values) = msg { + count(label_values) == 0 + msg := sprintf("Pod <%v> requests GPU resources but does not target nodes with label key <%v> using node affinity or nodeSelector", [input.review.object.metadata.name, label_key]) + } + + violation_message(label_key, label_values) = msg { + count(label_values) > 0 + msg := sprintf("Pod <%v> requests GPU resources but does not target nodes with label <%v> matching one of <%v> using node affinity or nodeSelector", [input.review.object.metadata.name, label_key, label_values]) + } + + pod_requests_gpu { + container := all_containers[_] + not is_exempt(container) + requests_gpu(container) + } + + all_containers[c] { + c := input.review.object.spec.containers[_] + } + + all_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + all_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + + requests_gpu(container) { + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu := limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + requests_gpu(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + gpu := requests["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_matching_node_selector(label_key) { + selector := input.review.object.spec.nodeSelector + value := selector[label_key] + value != "" + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) == 0 + } + + has_matching_node_selector(label_key) { + selector := input.review.object.spec.nodeSelector + value := selector[label_key] + label_values := object.get(input.parameters, "nodeLabelValues", []) + label_values[_] == value + } + + has_matching_node_affinity(label_key) { + term := input.review.object.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[_] + expr := term.matchExpressions[_] + expr.key == label_key + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) == 0 + expr.operator == "Exists" + } + + has_matching_node_affinity(label_key) { + term := input.review.object.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[_] + expr := term.matchExpressions[_] + expr.key == label_key + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) > 0 + expr.operator == "In" + values := object.get(expr, "values", []) + count(values) > 0 + not has_disallowed_affinity_value(values, label_values) + } + + has_disallowed_affinity_value(values, label_values) { + value := values[_] + not allowed_affinity_value(value, label_values) + } + + allowed_affinity_value(value, label_values) { + label_values[_] == value + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } \ No newline at end of file diff --git a/library/general/gpuresourcelimits/kustomization.yaml b/library/general/gpuresourcelimits/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/library/general/gpuresourcelimits/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/constraint.yaml b/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/constraint.yaml new file mode 100644 index 000000000..73fd79687 --- /dev/null +++ b/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 diff --git a/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/example_disallowed.yaml b/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/example_disallowed.yaml new file mode 100644 index 000000000..956b583ff --- /dev/null +++ b/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exceeds-limit +spec: + containers: + - name: training + image: myrepo/large-training:v1 + resources: + limits: + nvidia.com/gpu: "8" diff --git a/library/general/gpuresourcelimits/samples/gpu-exempt-over-limit/constraint.yaml b/library/general/gpuresourcelimits/samples/gpu-exempt-over-limit/constraint.yaml new file mode 100644 index 000000000..6e87e1ef6 --- /dev/null +++ b/library/general/gpuresourcelimits/samples/gpu-exempt-over-limit/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/library/general/gpuresourcelimits/samples/gpu-exempt-over-limit/example_allowed.yaml b/library/general/gpuresourcelimits/samples/gpu-exempt-over-limit/example_allowed.yaml new file mode 100644 index 000000000..f25579b89 --- /dev/null +++ b/library/general/gpuresourcelimits/samples/gpu-exempt-over-limit/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-over-limit +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "8" \ No newline at end of file diff --git a/library/general/gpuresourcelimits/samples/gpu-max-disabled/constraint.yaml b/library/general/gpuresourcelimits/samples/gpu-max-disabled/constraint.yaml new file mode 100644 index 000000000..654cb75e8 --- /dev/null +++ b/library/general/gpuresourcelimits/samples/gpu-max-disabled/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpuresourcelimits/samples/gpu-max-disabled/example_allowed.yaml b/library/general/gpuresourcelimits/samples/gpu-max-disabled/example_allowed.yaml new file mode 100644 index 000000000..0d397154a --- /dev/null +++ b/library/general/gpuresourcelimits/samples/gpu-max-disabled/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-max-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "8" \ No newline at end of file diff --git a/library/general/gpuresourcelimits/samples/gpu-within-limit/constraint.yaml b/library/general/gpuresourcelimits/samples/gpu-within-limit/constraint.yaml new file mode 100644 index 000000000..73fd79687 --- /dev/null +++ b/library/general/gpuresourcelimits/samples/gpu-within-limit/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 diff --git a/library/general/gpuresourcelimits/samples/gpu-within-limit/example_allowed.yaml b/library/general/gpuresourcelimits/samples/gpu-within-limit/example_allowed.yaml new file mode 100644 index 000000000..37809a15b --- /dev/null +++ b/library/general/gpuresourcelimits/samples/gpu-within-limit/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-within-limit +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "2" diff --git a/library/general/gpuresourcelimits/samples/init-gpu-exceeds-limit/constraint.yaml b/library/general/gpuresourcelimits/samples/init-gpu-exceeds-limit/constraint.yaml new file mode 100644 index 000000000..fbb00baff --- /dev/null +++ b/library/general/gpuresourcelimits/samples/init-gpu-exceeds-limit/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 \ No newline at end of file diff --git a/library/general/gpuresourcelimits/samples/init-gpu-exceeds-limit/example_disallowed.yaml b/library/general/gpuresourcelimits/samples/init-gpu-exceeds-limit/example_disallowed.yaml new file mode 100644 index 000000000..5166b5320 --- /dev/null +++ b/library/general/gpuresourcelimits/samples/init-gpu-exceeds-limit/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: init-gpu-exceeds-limit +spec: + initContainers: + - name: setup + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "8" + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" \ No newline at end of file diff --git a/library/general/gpuresourcelimits/samples/non-gpu-pod/constraint.yaml b/library/general/gpuresourcelimits/samples/non-gpu-pod/constraint.yaml new file mode 100644 index 000000000..fbb00baff --- /dev/null +++ b/library/general/gpuresourcelimits/samples/non-gpu-pod/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 \ No newline at end of file diff --git a/library/general/gpuresourcelimits/samples/non-gpu-pod/example_allowed.yaml b/library/general/gpuresourcelimits/samples/non-gpu-pod/example_allowed.yaml new file mode 100644 index 000000000..492a70fdf --- /dev/null +++ b/library/general/gpuresourcelimits/samples/non-gpu-pod/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" \ No newline at end of file diff --git a/library/general/gpuresourcelimits/suite.yaml b/library/general/gpuresourcelimits/suite.yaml new file mode 100644 index 000000000..bf69ef012 --- /dev/null +++ b/library/general/gpuresourcelimits/suite.yaml @@ -0,0 +1,53 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpuresourcelimits +tests: +- name: gpu-within-limit + template: template.yaml + constraint: samples/gpu-within-limit/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-within-limit/example_allowed.yaml + assertions: + - violations: no +- name: gpu-exceeds-limit + template: template.yaml + constraint: samples/gpu-exceeds-limit/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-exceeds-limit/example_disallowed.yaml + assertions: + - violations: yes +- name: non-gpu-pod + template: template.yaml + constraint: samples/non-gpu-pod/constraint.yaml + cases: + - name: example-allowed + object: samples/non-gpu-pod/example_allowed.yaml + assertions: + - violations: no +- name: gpu-exempt-over-limit + template: template.yaml + constraint: samples/gpu-exempt-over-limit/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exempt-over-limit/example_allowed.yaml + assertions: + - violations: no +- name: gpu-max-disabled + template: template.yaml + constraint: samples/gpu-max-disabled/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-max-disabled/example_allowed.yaml + assertions: + - violations: no +- name: init-gpu-exceeds-limit + template: template.yaml + constraint: samples/init-gpu-exceeds-limit/constraint.yaml + cases: + - name: example-disallowed + object: samples/init-gpu-exceeds-limit/example_disallowed.yaml + assertions: + - violations: yes diff --git a/library/general/gpuresourcelimits/template.yaml b/library/general/gpuresourcelimits/template.yaml new file mode 100644 index 000000000..299552859 --- /dev/null +++ b/library/general/gpuresourcelimits/template.yaml @@ -0,0 +1,126 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuresourcelimits + annotations: + metadata.gatekeeper.sh/title: "GPU Resource Limits" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Enforces a maximum number of NVIDIA GPUs (nvidia.com/gpu) that a single + container may request. This prevents individual containers from hoarding + GPU resources on shared clusters, particularly for AI/ML training workloads. +spec: + crd: + spec: + names: + kind: K8sGpuResourceLimits + validation: + openAPIV3Schema: + type: object + description: >- + Enforces a maximum number of NVIDIA GPUs per container. + properties: + maxGpuPerContainer: + description: >- + The maximum number of GPUs a single container may request. + type: integer + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: maxGpu + expression: 'has(variables.params.maxGpuPerContainer) ? variables.params.maxGpuPerContainer : 0' + - name: badContainers + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + variables.maxGpu > 0 && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity(string(variables.maxGpu))) > 0 + ).map(container, "Container <" + container.name + "> requests " + string(container.resources.limits["nvidia.com/gpu"]) + " GPUs, which exceeds the maximum allowed " + string(variables.maxGpu)) + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpuresourcelimits + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input_containers[_] + not is_exempt(container) + gpu_count := to_number(container.resources.limits["nvidia.com/gpu"]) + gpu_count > 0 + max_gpu := object.get(input, ["parameters", "maxGpuPerContainer"], 0) + max_gpu > 0 + gpu_count > max_gpu + msg := sprintf("Container <%v> requests %v GPUs, which exceeds the maximum allowed %v", [container.name, gpu_count, max_gpu]) + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/library/general/gpusharedmemory/kustomization.yaml b/library/general/gpusharedmemory/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/library/general/gpusharedmemory/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/library/general/gpusharedmemory/samples/gpu-exempt-without-shm/constraint.yaml b/library/general/gpusharedmemory/samples/gpu-exempt-without-shm/constraint.yaml new file mode 100644 index 000000000..498b77964 --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-exempt-without-shm/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/library/general/gpusharedmemory/samples/gpu-exempt-without-shm/example_allowed.yaml b/library/general/gpusharedmemory/samples/gpu-exempt-without-shm/example_allowed.yaml new file mode 100644 index 000000000..d839d7e0d --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-exempt-without-shm/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-shm +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpusharedmemory/samples/gpu-shm-non-memory/constraint.yaml b/library/general/gpusharedmemory/samples/gpu-shm-non-memory/constraint.yaml new file mode 100644 index 000000000..f8d2ad123 --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-shm-non-memory/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpusharedmemory/samples/gpu-shm-non-memory/example_disallowed.yaml b/library/general/gpusharedmemory/samples/gpu-shm-non-memory/example_disallowed.yaml new file mode 100644 index 000000000..b3a244b79 --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-shm-non-memory/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-shm-non-memory +spec: + volumes: + - name: dshm + emptyDir: {} + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /dev/shm \ No newline at end of file diff --git a/library/general/gpusharedmemory/samples/gpu-shm-wrong-path/constraint.yaml b/library/general/gpusharedmemory/samples/gpu-shm-wrong-path/constraint.yaml new file mode 100644 index 000000000..f8d2ad123 --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-shm-wrong-path/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpusharedmemory/samples/gpu-shm-wrong-path/example_disallowed.yaml b/library/general/gpusharedmemory/samples/gpu-shm-wrong-path/example_disallowed.yaml new file mode 100644 index 000000000..5a09f03b2 --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-shm-wrong-path/example_disallowed.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-shm-wrong-path +spec: + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /cache \ No newline at end of file diff --git a/library/general/gpusharedmemory/samples/gpu-with-shm/constraint.yaml b/library/general/gpusharedmemory/samples/gpu-with-shm/constraint.yaml new file mode 100644 index 000000000..3ebae67a8 --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-with-shm/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/library/general/gpusharedmemory/samples/gpu-with-shm/example_allowed.yaml b/library/general/gpusharedmemory/samples/gpu-with-shm/example_allowed.yaml new file mode 100644 index 000000000..978bdeeab --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-with-shm/example_allowed.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-shm +spec: + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 8Gi + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /dev/shm diff --git a/library/general/gpusharedmemory/samples/gpu-without-shm/constraint.yaml b/library/general/gpusharedmemory/samples/gpu-without-shm/constraint.yaml new file mode 100644 index 000000000..3ebae67a8 --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-without-shm/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/library/general/gpusharedmemory/samples/gpu-without-shm/example_disallowed.yaml b/library/general/gpusharedmemory/samples/gpu-without-shm/example_disallowed.yaml new file mode 100644 index 000000000..014e92d27 --- /dev/null +++ b/library/general/gpusharedmemory/samples/gpu-without-shm/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-shm +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/library/general/gpusharedmemory/samples/no-gpu/constraint.yaml b/library/general/gpusharedmemory/samples/no-gpu/constraint.yaml new file mode 100644 index 000000000..3ebae67a8 --- /dev/null +++ b/library/general/gpusharedmemory/samples/no-gpu/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/library/general/gpusharedmemory/samples/no-gpu/example_allowed.yaml b/library/general/gpusharedmemory/samples/no-gpu/example_allowed.yaml new file mode 100644 index 000000000..2fc1db2ad --- /dev/null +++ b/library/general/gpusharedmemory/samples/no-gpu/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/library/general/gpusharedmemory/suite.yaml b/library/general/gpusharedmemory/suite.yaml new file mode 100644 index 000000000..f30b1cadf --- /dev/null +++ b/library/general/gpusharedmemory/suite.yaml @@ -0,0 +1,53 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpusharedmemory +tests: +- name: gpu-with-shm + template: template.yaml + constraint: samples/gpu-with-shm/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-with-shm/example_allowed.yaml + assertions: + - violations: no +- name: gpu-without-shm + template: template.yaml + constraint: samples/gpu-without-shm/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-without-shm/example_disallowed.yaml + assertions: + - violations: yes +- name: no-gpu + template: template.yaml + constraint: samples/no-gpu/constraint.yaml + cases: + - name: example-allowed + object: samples/no-gpu/example_allowed.yaml + assertions: + - violations: no +- name: gpu-shm-non-memory + template: template.yaml + constraint: samples/gpu-shm-non-memory/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-shm-non-memory/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-shm-wrong-path + template: template.yaml + constraint: samples/gpu-shm-wrong-path/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-shm-wrong-path/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-exempt-without-shm + template: template.yaml + constraint: samples/gpu-exempt-without-shm/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exempt-without-shm/example_allowed.yaml + assertions: + - violations: no diff --git a/library/general/gpusharedmemory/template.yaml b/library/general/gpusharedmemory/template.yaml new file mode 100644 index 000000000..a6cc2b545 --- /dev/null +++ b/library/general/gpusharedmemory/template.yaml @@ -0,0 +1,126 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpusharedmemory + annotations: + metadata.gatekeeper.sh/title: "GPU Shared Memory Required" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-ai-training-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to mount a + memory-backed emptyDir volume at /dev/shm. PyTorch DataLoader, NCCL + multi-GPU communication, and most training frameworks require shared memory + beyond the default 64MB. +spec: + crd: + spec: + names: + kind: K8sGpuSharedMemory + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to mount a memory-backed volume at /dev/shm. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.containers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: volumes + expression: 'has(variables.anyObject.spec.volumes) ? variables.anyObject.spec.volumes : []' + - name: memoryVolNames + expression: | + variables.volumes.filter(v, + has(v.emptyDir) && has(v.emptyDir.medium) && v.emptyDir.medium == "Memory" + ).map(v, v.name) + - name: badContainers + expression: | + variables.containers.filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + (!has(container.volumeMounts) || + !container.volumeMounts.exists(vm, + vm.mountPath == "/dev/shm" && + vm.name in variables.memoryVolNames + ) + ) + ).map(container, "Container <" + container.name + "> requests GPU resources but does not mount a memory-backed volume at /dev/shm") + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpusharedmemory + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input.review.object.spec.containers[_] + not is_exempt(container) + has_gpu_request(container) + not has_shm_mount(container) + msg := sprintf("Container <%v> requests GPU resources but does not mount a memory-backed volume at /dev/shm", [container.name]) + } + + has_gpu_request(container) { + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_shm_mount(container) { + mount := container.volumeMounts[_] + mount.mountPath == "/dev/shm" + volume := input.review.object.spec.volumes[_] + volume.name == mount.name + volume.emptyDir.medium == "Memory" + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/library/general/gpuworkloadresources/kustomization.yaml b/library/general/gpuworkloadresources/kustomization.yaml new file mode 100644 index 000000000..24dedaea1 --- /dev/null +++ b/library/general/gpuworkloadresources/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-compliant/constraint.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-compliant/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-compliant/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-compliant/example_allowed.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-compliant/example_allowed.yaml new file mode 100644 index 000000000..ff4ab90fd --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-compliant/example_allowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-compliant +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-cpu-request-missing/constraint.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-cpu-request-missing/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-cpu-request-missing/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-cpu-request-missing/example_disallowed.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-cpu-request-missing/example_disallowed.yaml new file mode 100644 index 000000000..b08908bfa --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-cpu-request-missing/example_disallowed.yaml @@ -0,0 +1,24 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-cpu-request-missing +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" + - name: logger + image: busybox:1.36 + resources: + requests: + memory: "256Mi" + limits: + memory: "256Mi" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-exempt/constraint.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-exempt/constraint.yaml new file mode 100644 index 000000000..4fdae6b96 --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-exempt/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-exempt/example_allowed.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-exempt/example_allowed.yaml new file mode 100644 index 000000000..9f8e7c5af --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-exempt/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-gpu-mismatch/constraint.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-gpu-mismatch/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-gpu-mismatch/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-gpu-mismatch/example_disallowed.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-gpu-mismatch/example_disallowed.yaml new file mode 100644 index 000000000..a9a0cf9b6 --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-gpu-mismatch/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-gpu-mismatch +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "2" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-gpu-request-missing/constraint.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-gpu-request-missing/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-gpu-request-missing/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-gpu-request-missing/example_gator_disallowed.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-gpu-request-missing/example_gator_disallowed.yaml new file mode 100644 index 000000000..3ee8d1b6d --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-gpu-request-missing/example_gator_disallowed.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-gpu-request-missing +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-memory-equivalent/constraint.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-memory-equivalent/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-memory-equivalent/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-memory-equivalent/example_allowed.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-memory-equivalent/example_allowed.yaml new file mode 100644 index 000000000..5fe5a904d --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-memory-equivalent/example_allowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-memory-equivalent +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "1024Mi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "1Gi" + cpu: "4" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-memory-mismatch/constraint.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-memory-mismatch/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-memory-mismatch/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/gpu-pod-memory-mismatch/example_disallowed.yaml b/library/general/gpuworkloadresources/samples/gpu-pod-memory-mismatch/example_disallowed.yaml new file mode 100644 index 000000000..a513a8bc8 --- /dev/null +++ b/library/general/gpuworkloadresources/samples/gpu-pod-memory-mismatch/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-memory-mismatch +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "8Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/non-gpu-pod/constraint.yaml b/library/general/gpuworkloadresources/samples/non-gpu-pod/constraint.yaml new file mode 100644 index 000000000..2e038b6bf --- /dev/null +++ b/library/general/gpuworkloadresources/samples/non-gpu-pod/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/gpuworkloadresources/samples/non-gpu-pod/example_allowed.yaml b/library/general/gpuworkloadresources/samples/non-gpu-pod/example_allowed.yaml new file mode 100644 index 000000000..fdf3a64ba --- /dev/null +++ b/library/general/gpuworkloadresources/samples/non-gpu-pod/example_allowed.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" \ No newline at end of file diff --git a/library/general/gpuworkloadresources/suite.yaml b/library/general/gpuworkloadresources/suite.yaml new file mode 100644 index 000000000..0173bd0d0 --- /dev/null +++ b/library/general/gpuworkloadresources/suite.yaml @@ -0,0 +1,69 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: gpuworkloadresources +tests: +- name: gpu-pod-compliant + template: template.yaml + constraint: samples/gpu-pod-compliant/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-compliant/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-memory-mismatch + template: template.yaml + constraint: samples/gpu-pod-memory-mismatch/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-memory-mismatch/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-pod-cpu-request-missing + template: template.yaml + constraint: samples/gpu-pod-cpu-request-missing/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-cpu-request-missing/example_disallowed.yaml + assertions: + - violations: yes +- name: non-gpu-pod + template: template.yaml + constraint: samples/non-gpu-pod/constraint.yaml + cases: + - name: example-allowed + object: samples/non-gpu-pod/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-gpu-request-missing + template: template.yaml + constraint: samples/gpu-pod-gpu-request-missing/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-gpu-request-missing/example_gator_disallowed.yaml + assertions: + - violations: yes +- name: gpu-pod-gpu-mismatch + template: template.yaml + constraint: samples/gpu-pod-gpu-mismatch/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-pod-gpu-mismatch/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-pod-memory-equivalent + template: template.yaml + constraint: samples/gpu-pod-memory-equivalent/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-memory-equivalent/example_allowed.yaml + assertions: + - violations: no +- name: gpu-pod-exempt + template: template.yaml + constraint: samples/gpu-pod-exempt/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-pod-exempt/example_allowed.yaml + assertions: + - violations: no \ No newline at end of file diff --git a/library/general/gpuworkloadresources/template.yaml b/library/general/gpuworkloadresources/template.yaml new file mode 100644 index 000000000..36c44d29b --- /dev/null +++ b/library/general/gpuworkloadresources/template.yaml @@ -0,0 +1,341 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuworkloadresources + annotations: + metadata.gatekeeper.sh/title: "GPU Workload Resources" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set + memory requests equal to limits for all non-exempt containers and to set + a CPU request. Containers that request GPUs must also set matching GPU + requests and limits. This keeps GPU workloads schedulable and predictable + while still allowing CPU burst above request. +spec: + crd: + spec: + names: + kind: K8sGpuWorkloadResources + validation: + openAPIV3Schema: + type: object + description: >- + Enforces memory request/limit alignment and cpu requests for GPU pods, + plus matching gpu requests and limits on GPU containers. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: enforcedContainers + expression: 'variables.containers + variables.initContainers' + - name: allContainers + expression: 'variables.enforcedContainers + variables.ephemeralContainers' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.allContainers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: gpuContainers + expression: | + variables.allContainers.filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + ( + (has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) || + (has(container.resources.requests) && + "nvidia.com/gpu" in container.resources.requests && + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) + ) + ) + - name: podRequestsGpu + expression: 'size(variables.gpuContainers) > 0' + - name: gpuRequestLimitMessages + expression: | + variables.gpuContainers.filter(container, + !has(container.resources.requests) || + !("nvidia.com/gpu" in container.resources.requests) || + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) <= 0 || + !has(container.resources.limits) || + !("nvidia.com/gpu" in container.resources.limits) || + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) <= 0 || + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity(string(container.resources.limits["nvidia.com/gpu"]))) != 0 + ).map(container, "Container <" + container.name + "> must set nvidia.com/gpu request equal to limit") + - name: memoryRequestLimitMessages + expression: | + !variables.podRequestsGpu ? [] : + variables.enforcedContainers.filter(container, + !(container.image in variables.exemptImages) && + ( + !has(container.resources) || + !has(container.resources.requests) || + !("memory" in container.resources.requests) || + !has(container.resources.limits) || + !("memory" in container.resources.limits) || + quantity(string(container.resources.requests["memory"])).compareTo(quantity(string(container.resources.limits["memory"]))) != 0 + ) + ).map(container, "Container <" + container.name + "> in a GPU pod must set memory request equal to limit") + - name: cpuRequestMessages + expression: | + !variables.podRequestsGpu ? [] : + variables.enforcedContainers.filter(container, + !(container.image in variables.exemptImages) && + ( + !has(container.resources) || + !has(container.resources.requests) || + !("cpu" in container.resources.requests) || + string(container.resources.requests["cpu"]) == "" + ) + ).map(container, "Container <" + container.name + "> in a GPU pod must set a cpu request") + validations: + - expression: 'size(variables.gpuRequestLimitMessages) == 0' + messageExpression: 'variables.gpuRequestLimitMessages.join(", ")' + - expression: 'size(variables.memoryRequestLimitMessages) == 0' + messageExpression: 'variables.memoryRequestLimitMessages.join(", ")' + - expression: 'size(variables.cpuRequestMessages) == 0' + messageExpression: 'variables.cpuRequestMessages.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpuworkloadresources + + import data.lib.exempt_container.is_exempt + + missing(obj, field) = true { + not obj[field] + } + + missing(obj, field) = true { + obj[field] == "" + } + + # 10 ** 21 + mem_multiple("E") = 1000000000000000000000 { true } + + # 10 ** 18 + mem_multiple("P") = 1000000000000000000 { true } + + # 10 ** 15 + mem_multiple("T") = 1000000000000000 { true } + + # 10 ** 12 + mem_multiple("G") = 1000000000000 { true } + + # 10 ** 9 + mem_multiple("M") = 1000000000 { true } + + # 10 ** 6 + mem_multiple("k") = 1000000 { true } + + # 10 ** 3 + mem_multiple("") = 1000 { true } + + # Kubernetes accepts millibyte precision when it probably shouldn't. + # https://github.com/kubernetes/kubernetes/issues/28741 + # 10 ** 0 + mem_multiple("m") = 1 { true } + + # 1000 * 2 ** 10 + mem_multiple("Ki") = 1024000 { true } + + # 1000 * 2 ** 20 + mem_multiple("Mi") = 1048576000 { true } + + # 1000 * 2 ** 30 + mem_multiple("Gi") = 1073741824000 { true } + + # 1000 * 2 ** 40 + mem_multiple("Ti") = 1099511627776000 { true } + + # 1000 * 2 ** 50 + mem_multiple("Pi") = 1125899906842624000 { true } + + # 1000 * 2 ** 60 + mem_multiple("Ei") = 1152921504606846976000 { true } + + get_suffix(mem) = suffix { + not is_string(mem) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 0 + suffix := substring(mem, count(mem) - 1, -1) + mem_multiple(suffix) + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 1 + suffix := substring(mem, count(mem) - 2, -1) + mem_multiple(suffix) + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 1 + not mem_multiple(substring(mem, count(mem) - 1, -1)) + not mem_multiple(substring(mem, count(mem) - 2, -1)) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) == 1 + not mem_multiple(substring(mem, count(mem) - 1, -1)) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) == 0 + suffix := "" + } + + canonify_mem(orig) = new { + is_number(orig) + new := orig * 1000 + } + + canonify_mem(orig) = new { + not is_number(orig) + suffix := get_suffix(orig) + raw := replace(orig, suffix, "") + regex.match("^[0-9]+(\\.[0-9]+)?$", raw) + new := to_number(raw) * mem_multiple(suffix) + } + + violation[{"msg": msg}] { + container := gpu_containers[_] + not has_matching_gpu_request_and_limit(container) + msg := sprintf("Container <%v> must set nvidia.com/gpu request equal to limit", [container.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + container := enforced_containers[_] + not is_exempt(container) + not has_matching_memory_request_and_limit(container) + msg := sprintf("Container <%v> in a GPU pod must set memory request equal to limit", [container.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + container := enforced_containers[_] + not is_exempt(container) + not has_cpu_request(container) + msg := sprintf("Container <%v> in a GPU pod must set a cpu request", [container.name]) + } + + pod_requests_gpu { + gpu_containers[_] + } + + gpu_containers[c] { + c := all_containers[_] + not is_exempt(c) + requests_gpu(c) + } + + enforced_containers[c] { + c := input.review.object.spec.containers[_] + } + + enforced_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + all_containers[c] { + c := enforced_containers[_] + } + + all_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + + requests_gpu(container) { + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu := limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + requests_gpu(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + gpu := requests["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_matching_gpu_request_and_limit(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu_request := requests["nvidia.com/gpu"] + gpu_limit := limits["nvidia.com/gpu"] + to_number(gpu_request) > 0 + to_number(gpu_limit) > 0 + to_number(gpu_request) == to_number(gpu_limit) + } + + has_matching_memory_request_and_limit(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + limits := object.get(object.get(container, "resources", {}), "limits", {}) + mem_request := requests["memory"] + mem_limit := limits["memory"] + canonify_mem(mem_request) == canonify_mem(mem_limit) + } + + has_cpu_request(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + not missing(requests, "cpu") + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } \ No newline at end of file diff --git a/library/general/kustomization.yaml b/library/general/kustomization.yaml index 6223d8f45..55eba5350 100644 --- a/library/general/kustomization.yaml +++ b/library/general/kustomization.yaml @@ -31,3 +31,11 @@ resources: - verifydeprecatedapi - storageclass - allowedreposv2 +- requiredgpuruntimeclass +- gpuactivedeadline +- gpunodetargeting +- nounsupportedgpu +- gpuworkloadresources +- requiredgputoleration +- gpuresourcelimits +- gpusharedmemory diff --git a/library/general/nounsupportedgpu/kustomization.yaml b/library/general/nounsupportedgpu/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/library/general/nounsupportedgpu/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/library/general/nounsupportedgpu/samples/gpu-ephemeral-without-env-var/constraint.yaml b/library/general/nounsupportedgpu/samples/gpu-ephemeral-without-env-var/constraint.yaml new file mode 100644 index 000000000..1bf71de18 --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-ephemeral-without-env-var/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/nounsupportedgpu/samples/gpu-ephemeral-without-env-var/example_disallowed.yaml b/library/general/nounsupportedgpu/samples/gpu-ephemeral-without-env-var/example_disallowed.yaml new file mode 100644 index 000000000..1416a5210 --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-ephemeral-without-env-var/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-ephemeral-without-env-var +spec: + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" + ephemeralContainers: + - name: debug-gpu + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/nounsupportedgpu/samples/gpu-exact-exempt/constraint.yaml b/library/general/nounsupportedgpu/samples/gpu-exact-exempt/constraint.yaml new file mode 100644 index 000000000..ea1935cd4 --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-exact-exempt/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:3.1.7" \ No newline at end of file diff --git a/library/general/nounsupportedgpu/samples/gpu-exact-exempt/example_allowed.yaml b/library/general/nounsupportedgpu/samples/gpu-exact-exempt/example_allowed.yaml new file mode 100644 index 000000000..775dcb40a --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-exact-exempt/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exact-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/nounsupportedgpu/samples/gpu-init-without-env-var/constraint.yaml b/library/general/nounsupportedgpu/samples/gpu-init-without-env-var/constraint.yaml new file mode 100644 index 000000000..1bf71de18 --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-init-without-env-var/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/nounsupportedgpu/samples/gpu-init-without-env-var/example_disallowed.yaml b/library/general/nounsupportedgpu/samples/gpu-init-without-env-var/example_disallowed.yaml new file mode 100644 index 000000000..69149470a --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-init-without-env-var/example_disallowed.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-init-without-env-var +spec: + initContainers: + - name: setup + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" \ No newline at end of file diff --git a/library/general/nounsupportedgpu/samples/gpu-with-env-var/constraint.yaml b/library/general/nounsupportedgpu/samples/gpu-with-env-var/constraint.yaml new file mode 100644 index 000000000..1fb10c208 --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-with-env-var/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/library/general/nounsupportedgpu/samples/gpu-with-env-var/example_allowed.yaml b/library/general/nounsupportedgpu/samples/gpu-with-env-var/example_allowed.yaml new file mode 100644 index 000000000..23e1bc9fd --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-with-env-var/example_allowed.yaml @@ -0,0 +1,14 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-allowed +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + env: + - name: NVIDIA_VISIBLE_DEVICES + value: all diff --git a/library/general/nounsupportedgpu/samples/gpu-without-env-var/constraint.yaml b/library/general/nounsupportedgpu/samples/gpu-without-env-var/constraint.yaml new file mode 100644 index 000000000..9e3d493ea --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-without-env-var/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" diff --git a/library/general/nounsupportedgpu/samples/gpu-without-env-var/example_allowed_exempt.yaml b/library/general/nounsupportedgpu/samples/gpu-without-env-var/example_allowed_exempt.yaml new file mode 100644 index 000000000..56b5d2c1a --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-without-env-var/example_allowed_exempt.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" diff --git a/library/general/nounsupportedgpu/samples/gpu-without-env-var/example_disallowed.yaml b/library/general/nounsupportedgpu/samples/gpu-without-env-var/example_disallowed.yaml new file mode 100644 index 000000000..f276fe228 --- /dev/null +++ b/library/general/nounsupportedgpu/samples/gpu-without-env-var/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-disallowed +spec: + containers: + - name: training + image: myrepo/custom-ml-image:v1 + resources: + limits: + nvidia.com/gpu: "1" diff --git a/library/general/nounsupportedgpu/samples/no-gpu-requested/constraint.yaml b/library/general/nounsupportedgpu/samples/no-gpu-requested/constraint.yaml new file mode 100644 index 000000000..1fb10c208 --- /dev/null +++ b/library/general/nounsupportedgpu/samples/no-gpu-requested/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] diff --git a/library/general/nounsupportedgpu/samples/no-gpu-requested/example_allowed.yaml b/library/general/nounsupportedgpu/samples/no-gpu-requested/example_allowed.yaml new file mode 100644 index 000000000..2fc1db2ad --- /dev/null +++ b/library/general/nounsupportedgpu/samples/no-gpu-requested/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/library/general/nounsupportedgpu/suite.yaml b/library/general/nounsupportedgpu/suite.yaml new file mode 100644 index 000000000..1b8b225ee --- /dev/null +++ b/library/general/nounsupportedgpu/suite.yaml @@ -0,0 +1,57 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: nounsupportedgpu +tests: +- name: gpu-with-env-var + template: template.yaml + constraint: samples/gpu-with-env-var/constraint.yaml + cases: + - name: example-allowed-gpu-with-env + object: samples/gpu-with-env-var/example_allowed.yaml + assertions: + - violations: no +- name: gpu-without-env-var + template: template.yaml + constraint: samples/gpu-without-env-var/constraint.yaml + cases: + - name: example-disallowed-gpu-no-env + object: samples/gpu-without-env-var/example_disallowed.yaml + assertions: + - violations: yes + - name: example-allowed-exempt-image + object: samples/gpu-without-env-var/example_allowed_exempt.yaml + assertions: + - violations: no +- name: no-gpu-requested + template: template.yaml + constraint: samples/no-gpu-requested/constraint.yaml + cases: + - name: example-allowed-no-gpu + object: samples/no-gpu-requested/example_allowed.yaml + assertions: + - violations: no +- name: gpu-init-without-env-var + template: template.yaml + constraint: samples/gpu-init-without-env-var/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-init-without-env-var/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-ephemeral-without-env-var + template: template.yaml + constraint: samples/gpu-ephemeral-without-env-var/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-ephemeral-without-env-var/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-exact-exempt + template: template.yaml + constraint: samples/gpu-exact-exempt/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exact-exempt/example_allowed.yaml + assertions: + - violations: no diff --git a/library/general/nounsupportedgpu/template.yaml b/library/general/nounsupportedgpu/template.yaml new file mode 100644 index 000000000..bd6ce987d --- /dev/null +++ b/library/general/nounsupportedgpu/template.yaml @@ -0,0 +1,130 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8snounsupportedgpu + annotations: + metadata.gatekeeper.sh/title: "No Unsupported GPU Requests" + metadata.gatekeeper.sh/version: 1.0.0 + description: >- + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set + the NVIDIA_VISIBLE_DEVICES environment variable, indicating the container + image is built to consume GPUs via the NVIDIA CUDA runtime. This prevents + GPU resource waste from containers that request GPUs but cannot use them. +spec: + crd: + spec: + names: + kind: K8sNoUnsupportedGpu + validation: + openAPIV3Schema: + type: object + description: >- + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set + the NVIDIA_VISIBLE_DEVICES environment variable. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: badContainers + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + (!has(container.env) || !container.env.exists(e, e.name == "NVIDIA_VISIBLE_DEVICES")) + ).map(container, "Container <" + container.name + "> requests nvidia.com/gpu but does not set the NVIDIA_VISIBLE_DEVICES environment variable") + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8snounsupportedgpu + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input_containers[_] + not is_exempt(container) + has_gpu_request(container) + not has_nvidia_env(container) + msg := sprintf("Container <%v> requests nvidia.com/gpu but does not set the NVIDIA_VISIBLE_DEVICES environment variable", [container.name]) + } + + has_gpu_request(container) { + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_nvidia_env(container) { + env := container.env[_] + env.name == "NVIDIA_VISIBLE_DEVICES" + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/library/general/requiredgpuruntimeclass/kustomization.yaml b/library/general/requiredgpuruntimeclass/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-exempt-without-runtimeclass/constraint.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-exempt-without-runtimeclass/constraint.yaml new file mode 100644 index 000000000..bc8bcf719 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-exempt-without-runtimeclass/constraint.yaml @@ -0,0 +1,14 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-exempt-without-runtimeclass/example_allowed.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-exempt-without-runtimeclass/example_allowed.yaml new file mode 100644 index 000000000..03c95f066 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-exempt-without-runtimeclass/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-runtimeclass +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-runtimeclass-disabled/constraint.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-runtimeclass-disabled/constraint.yaml new file mode 100644 index 000000000..b45ecf2a0 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-runtimeclass-disabled/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-runtimeclass-disabled/example_allowed.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-runtimeclass-disabled/example_allowed.yaml new file mode 100644 index 000000000..83d99b8fd --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-runtimeclass-disabled/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-runtimeclass-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-with-runtimeclass/constraint.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-with-runtimeclass/constraint.yaml new file mode 100644 index 000000000..2ba97b990 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-with-runtimeclass/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-with-runtimeclass/example_allowed.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-with-runtimeclass/example_allowed.yaml new file mode 100644 index 000000000..0f0111cd0 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-with-runtimeclass/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-runtime +spec: + runtimeClassName: nvidia + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-without-runtimeclass/constraint.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-without-runtimeclass/constraint.yaml new file mode 100644 index 000000000..2ba97b990 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-without-runtimeclass/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-without-runtimeclass/example_disallowed.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-without-runtimeclass/example_disallowed.yaml new file mode 100644 index 000000000..e98db8edd --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-without-runtimeclass/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-runtime +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-wrong-runtimeclass/constraint.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-wrong-runtimeclass/constraint.yaml new file mode 100644 index 000000000..fe1939500 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-wrong-runtimeclass/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" \ No newline at end of file diff --git a/library/general/requiredgpuruntimeclass/samples/gpu-wrong-runtimeclass/example_disallowed.yaml b/library/general/requiredgpuruntimeclass/samples/gpu-wrong-runtimeclass/example_disallowed.yaml new file mode 100644 index 000000000..d40249d32 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/gpu-wrong-runtimeclass/example_disallowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-wrong-runtimeclass +spec: + runtimeClassName: runc + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/requiredgpuruntimeclass/samples/no-gpu/constraint.yaml b/library/general/requiredgpuruntimeclass/samples/no-gpu/constraint.yaml new file mode 100644 index 000000000..2ba97b990 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/no-gpu/constraint.yaml @@ -0,0 +1,12 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" diff --git a/library/general/requiredgpuruntimeclass/samples/no-gpu/example_allowed.yaml b/library/general/requiredgpuruntimeclass/samples/no-gpu/example_allowed.yaml new file mode 100644 index 000000000..2fc1db2ad --- /dev/null +++ b/library/general/requiredgpuruntimeclass/samples/no-gpu/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/library/general/requiredgpuruntimeclass/suite.yaml b/library/general/requiredgpuruntimeclass/suite.yaml new file mode 100644 index 000000000..9eec23598 --- /dev/null +++ b/library/general/requiredgpuruntimeclass/suite.yaml @@ -0,0 +1,53 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: requiredgpuruntimeclass +tests: +- name: gpu-with-runtimeclass + template: template.yaml + constraint: samples/gpu-with-runtimeclass/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-with-runtimeclass/example_allowed.yaml + assertions: + - violations: no +- name: gpu-without-runtimeclass + template: template.yaml + constraint: samples/gpu-without-runtimeclass/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-without-runtimeclass/example_disallowed.yaml + assertions: + - violations: yes +- name: no-gpu + template: template.yaml + constraint: samples/no-gpu/constraint.yaml + cases: + - name: example-allowed + object: samples/no-gpu/example_allowed.yaml + assertions: + - violations: no +- name: gpu-wrong-runtimeclass + template: template.yaml + constraint: samples/gpu-wrong-runtimeclass/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-wrong-runtimeclass/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-runtimeclass-disabled + template: template.yaml + constraint: samples/gpu-runtimeclass-disabled/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-runtimeclass-disabled/example_allowed.yaml + assertions: + - violations: no +- name: gpu-exempt-without-runtimeclass + template: template.yaml + constraint: samples/gpu-exempt-without-runtimeclass/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exempt-without-runtimeclass/example_allowed.yaml + assertions: + - violations: no diff --git a/library/general/requiredgpuruntimeclass/template.yaml b/library/general/requiredgpuruntimeclass/template.yaml new file mode 100644 index 000000000..452ba13eb --- /dev/null +++ b/library/general/requiredgpuruntimeclass/template.yaml @@ -0,0 +1,139 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8srequiredgpuruntimeclass + annotations: + metadata.gatekeeper.sh/title: "Required GPU Runtime Class" + metadata.gatekeeper.sh/version: 1.0.0 + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to specify + a runtimeClassName from an allowed list. This ensures GPU workloads use + the proper container runtime (e.g., nvidia) rather than relying on default + runtime hooks. +spec: + crd: + spec: + names: + kind: K8sRequiredGpuRuntimeClass + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to specify an allowed runtimeClassName. + properties: + allowedRuntimeClassNames: + description: >- + List of allowed runtime class names for GPU workloads (e.g., ["nvidia"]). + type: array + items: + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: allowedRuntimeClassNames + expression: 'has(variables.params.allowedRuntimeClassNames) ? variables.params.allowedRuntimeClassNames : []' + - name: hasAllowedRc + expression: | + !has(variables.anyObject.spec.runtimeClassName) ? false : + variables.anyObject.spec.runtimeClassName in variables.allowedRuntimeClassNames + validations: + - expression: '!variables.podRequestsGpu || size(variables.allowedRuntimeClassNames) == 0 || variables.hasAllowedRc' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not specify an allowed runtimeClassName (allowed: " + variables.allowedRuntimeClassNames.join(", ") + ")"' + - engine: Rego + source: + rego: | + package k8srequiredgpuruntimeclass + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + allowed := object.get(input, ["parameters", "allowedRuntimeClassNames"], []) + count(allowed) > 0 + not has_allowed_runtime_class(allowed) + msg := sprintf("Pod <%v> requests GPU resources but does not specify an allowed runtimeClassName (allowed: %v)", [input.review.object.metadata.name, allowed]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_allowed_runtime_class(allowed) { + rc := input.review.object.spec.runtimeClassName + rc == allowed[_] + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/library/general/requiredgputoleration/kustomization.yaml b/library/general/requiredgputoleration/kustomization.yaml new file mode 100644 index 000000000..7d70d11b7 --- /dev/null +++ b/library/general/requiredgputoleration/kustomization.yaml @@ -0,0 +1,2 @@ +resources: + - template.yaml diff --git a/library/general/requiredgputoleration/samples/gpu-exempt-without-toleration/constraint.yaml b/library/general/requiredgputoleration/samples/gpu-exempt-without-toleration/constraint.yaml new file mode 100644 index 000000000..153d430e7 --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-exempt-without-toleration/constraint.yaml @@ -0,0 +1,13 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" + exemptImages: + - "nvidia/dcgm-exporter:*" \ No newline at end of file diff --git a/library/general/requiredgputoleration/samples/gpu-exempt-without-toleration/example_allowed.yaml b/library/general/requiredgputoleration/samples/gpu-exempt-without-toleration/example_allowed.yaml new file mode 100644 index 000000000..71d46d9f0 --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-exempt-without-toleration/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-toleration +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/requiredgputoleration/samples/gpu-toleration-disabled/constraint.yaml b/library/general/requiredgputoleration/samples/gpu-toleration-disabled/constraint.yaml new file mode 100644 index 000000000..6e3a75317 --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-toleration-disabled/constraint.yaml @@ -0,0 +1,9 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] \ No newline at end of file diff --git a/library/general/requiredgputoleration/samples/gpu-toleration-disabled/example_allowed.yaml b/library/general/requiredgputoleration/samples/gpu-toleration-disabled/example_allowed.yaml new file mode 100644 index 000000000..485ae9a4b --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-toleration-disabled/example_allowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-toleration-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" \ No newline at end of file diff --git a/library/general/requiredgputoleration/samples/gpu-toleration-key-only/constraint.yaml b/library/general/requiredgputoleration/samples/gpu-toleration-key-only/constraint.yaml new file mode 100644 index 000000000..9bce00a68 --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-toleration-key-only/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" \ No newline at end of file diff --git a/library/general/requiredgputoleration/samples/gpu-toleration-key-only/example_allowed.yaml b/library/general/requiredgputoleration/samples/gpu-toleration-key-only/example_allowed.yaml new file mode 100644 index 000000000..1a992fcaa --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-toleration-key-only/example_allowed.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-toleration-key-only +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "present" + effect: "NoExecute" \ No newline at end of file diff --git a/library/general/requiredgputoleration/samples/gpu-with-toleration/constraint.yaml b/library/general/requiredgputoleration/samples/gpu-with-toleration/constraint.yaml new file mode 100644 index 000000000..ba4ce180c --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-with-toleration/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" diff --git a/library/general/requiredgputoleration/samples/gpu-with-toleration/example_allowed.yaml b/library/general/requiredgputoleration/samples/gpu-with-toleration/example_allowed.yaml new file mode 100644 index 000000000..46a3d0af3 --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-with-toleration/example_allowed.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" diff --git a/library/general/requiredgputoleration/samples/gpu-without-toleration/constraint.yaml b/library/general/requiredgputoleration/samples/gpu-without-toleration/constraint.yaml new file mode 100644 index 000000000..ba4ce180c --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-without-toleration/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" diff --git a/library/general/requiredgputoleration/samples/gpu-without-toleration/example_disallowed.yaml b/library/general/requiredgputoleration/samples/gpu-without-toleration/example_disallowed.yaml new file mode 100644 index 000000000..9cd325882 --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-without-toleration/example_disallowed.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" diff --git a/library/general/requiredgputoleration/samples/gpu-wrong-toleration/constraint.yaml b/library/general/requiredgputoleration/samples/gpu-wrong-toleration/constraint.yaml new file mode 100644 index 000000000..9bce00a68 --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-wrong-toleration/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" \ No newline at end of file diff --git a/library/general/requiredgputoleration/samples/gpu-wrong-toleration/example_disallowed.yaml b/library/general/requiredgputoleration/samples/gpu-wrong-toleration/example_disallowed.yaml new file mode 100644 index 000000000..3d866fa43 --- /dev/null +++ b/library/general/requiredgputoleration/samples/gpu-wrong-toleration/example_disallowed.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: gpu-wrong-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "workload" + operator: "Exists" + effect: "NoSchedule" \ No newline at end of file diff --git a/library/general/requiredgputoleration/samples/no-gpu/constraint.yaml b/library/general/requiredgputoleration/samples/no-gpu/constraint.yaml new file mode 100644 index 000000000..ba4ce180c --- /dev/null +++ b/library/general/requiredgputoleration/samples/no-gpu/constraint.yaml @@ -0,0 +1,11 @@ +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" diff --git a/library/general/requiredgputoleration/samples/no-gpu/example_allowed.yaml b/library/general/requiredgputoleration/samples/no-gpu/example_allowed.yaml new file mode 100644 index 000000000..2fc1db2ad --- /dev/null +++ b/library/general/requiredgputoleration/samples/no-gpu/example_allowed.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" diff --git a/library/general/requiredgputoleration/suite.yaml b/library/general/requiredgputoleration/suite.yaml new file mode 100644 index 000000000..d53dd3af7 --- /dev/null +++ b/library/general/requiredgputoleration/suite.yaml @@ -0,0 +1,61 @@ +kind: Suite +apiVersion: test.gatekeeper.sh/v1alpha1 +metadata: + name: requiredgputoleration +tests: +- name: gpu-with-toleration + template: template.yaml + constraint: samples/gpu-with-toleration/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-with-toleration/example_allowed.yaml + assertions: + - violations: no +- name: gpu-without-toleration + template: template.yaml + constraint: samples/gpu-without-toleration/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-without-toleration/example_disallowed.yaml + assertions: + - violations: yes +- name: no-gpu + template: template.yaml + constraint: samples/no-gpu/constraint.yaml + cases: + - name: example-allowed + object: samples/no-gpu/example_allowed.yaml + assertions: + - violations: no +- name: gpu-wrong-toleration + template: template.yaml + constraint: samples/gpu-wrong-toleration/constraint.yaml + cases: + - name: example-disallowed + object: samples/gpu-wrong-toleration/example_disallowed.yaml + assertions: + - violations: yes +- name: gpu-toleration-disabled + template: template.yaml + constraint: samples/gpu-toleration-disabled/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-toleration-disabled/example_allowed.yaml + assertions: + - violations: no +- name: gpu-exempt-without-toleration + template: template.yaml + constraint: samples/gpu-exempt-without-toleration/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-exempt-without-toleration/example_allowed.yaml + assertions: + - violations: no +- name: gpu-toleration-key-only + template: template.yaml + constraint: samples/gpu-toleration-key-only/constraint.yaml + cases: + - name: example-allowed + object: samples/gpu-toleration-key-only/example_allowed.yaml + assertions: + - violations: no diff --git a/library/general/requiredgputoleration/template.yaml b/library/general/requiredgputoleration/template.yaml new file mode 100644 index 000000000..01baef028 --- /dev/null +++ b/library/general/requiredgputoleration/template.yaml @@ -0,0 +1,137 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8srequiredgputoleration + annotations: + metadata.gatekeeper.sh/title: "Required GPU Toleration" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to include + a toleration for the specified GPU node taint. This ensures GPU workloads + are properly configured to schedule on GPU nodes. +spec: + crd: + spec: + names: + kind: K8sRequiredGpuToleration + validation: + openAPIV3Schema: + type: object + description: >- + Requires pods requesting GPUs to include a specified toleration. + properties: + tolerationKey: + description: >- + The taint key that GPU pods must tolerate (e.g., "nvidia.com/gpu"). + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: tolerationKey + expression: 'has(variables.params.tolerationKey) ? variables.params.tolerationKey : ""' + - name: hasToleration + expression: | + !has(variables.anyObject.spec.tolerations) ? false : + variables.anyObject.spec.tolerations.exists(t, t.key == variables.tolerationKey) + validations: + - expression: '!variables.podRequestsGpu || variables.tolerationKey == "" || variables.hasToleration' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not tolerate taint key <" + variables.tolerationKey + ">"' + - engine: Rego + source: + rego: | + package k8srequiredgputoleration + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + toleration_key := object.get(input, ["parameters", "tolerationKey"], "") + toleration_key != "" + not has_toleration(toleration_key) + msg := sprintf("Pod <%v> requests GPU resources but does not tolerate taint key <%v>", [input.review.object.metadata.name, toleration_key]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_toleration(key) { + toleration := input.review.object.spec.tolerations[_] + toleration.key == key + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } diff --git a/scripts/website/generate.go b/scripts/website/generate.go index 842391c2f..2e2f25450 100644 --- a/scripts/website/generate.go +++ b/scripts/website/generate.go @@ -7,11 +7,11 @@ import ( "os" "path/filepath" "regexp" + "slices" "sort" "strings" "gopkg.in/yaml.v3" - k8sslices "k8s.io/utils/strings/slices" ) const ( @@ -26,6 +26,11 @@ const ( // regex patterns. pspReadmeLinkPattern = `\[([^\[\]]+)\]\(([^(]+)\)` + + // AI workload bundle names. + aiWorkloadInferenceBundle = "gatekeeper-ai-inference-policies" + aiWorkloadTrainingBundle = "gatekeeper-ai-training-policies" + aiWorkloadSafetyBundle = "gatekeeper-gpu-safety-policies" ) // Skip including examples for the following Kinds. @@ -150,6 +155,10 @@ func main() { examples := "" for _, testCase := range test.Cases { + if strings.Contains(filepath.Base(testCase.Object), "example_gator_") { + continue + } + exampleRawURL := sourceURL + filepath.Join(entryPoint, entry.Name(), dir.Name(), testCase.Object) exampleContent, err := os.ReadFile(filepath.Join(basePath, dir.Name(), testCase.Object)) @@ -168,7 +177,7 @@ func main() { if exampleKind, ok := exampleResource["kind"].(string); !ok { fmt.Printf("error while parsing kind: %v", exampleRawURL) panic(err) - } else if !k8sslices.Contains(skipExampleKinds, exampleKind) { + } else if !slices.Contains(skipExampleKinds, exampleKind) { examples += fmt.Sprintf("
\n%s\n\n```yaml\n%s\n```\n\nUsage\n\n```shell\nkubectl apply -f %s\n```\n\n
\n", testCase.Name, exampleContent, exampleRawURL) } } @@ -350,8 +359,40 @@ func main() { // update sidebar from template fmt.Println("Updating sidebar") + // Generate AI workload profile items for sidebar (policies organized by bundle). + // Policies appear in every profile they belong to, since safety, training, and + // inference bundles intentionally overlap. + aiWorkloadSafetyItemsList := generateSidebarItems(bundleItems[aiWorkloadSafetyBundle], "validation/", " ") + aiWorkloadTrainingItemsList := generateSidebarItems(bundleItems[aiWorkloadTrainingBundle], "validation/", " ") + aiWorkloadInferenceItemsList := generateSidebarItems(bundleItems[aiWorkloadInferenceBundle], "validation/", " ") + + allAIWorkloadItems := make(map[string]bool) + for _, bundleName := range []string{aiWorkloadSafetyBundle, aiWorkloadTrainingBundle, aiWorkloadInferenceBundle} { + for _, item := range bundleItems[bundleName] { + allAIWorkloadItems[item] = true + } + } + + var otherAIWorkloadItems []string + for _, item := range validationSidebarItems["general"] { + if isAIWorkloadItem(item) { + if !allAIWorkloadItems[item] { + otherAIWorkloadItems = append(otherAIWorkloadItems, item) + } + allAIWorkloadItems[item] = true + } + } + otherAIWorkloadItemsList := generateSidebarItems(otherAIWorkloadItems, "validation/", " ") + + var generalItems []string + for _, item := range validationSidebarItems["general"] { + if !allAIWorkloadItems[item] { + generalItems = append(generalItems, item) + } + } + // Generate General items - generalItemsList := generateSidebarItems(validationSidebarItems["general"], "validation/", " ") + generalItemsList := generateSidebarItems(generalItems, "validation/", " ") // Generate Mutation items mutationItemsList := generateSidebarItems(mutationSidebarItems["pod-security-policy"], "mutation-examples/", " ") @@ -389,6 +430,10 @@ func main() { sidebarReplacer := strings.NewReplacer( "%GENERAL_ITEMS%", generalItemsList, "%MUTATION_ITEMS%", mutationItemsList, + "%AI_WORKLOAD_SAFETY_ITEMS%", aiWorkloadSafetyItemsList, + "%AI_WORKLOAD_TRAINING_ITEMS%", aiWorkloadTrainingItemsList, + "%AI_WORKLOAD_INFERENCE_ITEMS%", aiWorkloadInferenceItemsList, + "%OTHER_AI_WORKLOAD_ITEMS%", otherAIWorkloadItemsList, "%BASELINE_ITEMS%", baselineItemsList, "%RESTRICTED_ITEMS%", restrictedItemsList, "%OTHER_PSP_ITEMS%", otherPSPItemsList, @@ -418,6 +463,10 @@ func generateSidebarItems(items []string, prefix string, indent string) string { return strings.Join(itemStrings, "\n") } +func isAIWorkloadItem(item string) bool { + return strings.Contains(strings.ToLower(item), "gpu") +} + // TODO: Use shared pkg. func getConstraintTemplateMetadata(constraintTemplate map[string]interface{}) map[string]interface{} { metadata, ok := constraintTemplate["metadata"].(map[string]interface{}) diff --git a/scripts/website/go.mod b/scripts/website/go.mod index 902a0f8fb..13a01f19f 100644 --- a/scripts/website/go.mod +++ b/scripts/website/go.mod @@ -1,5 +1,7 @@ module gatekeeper-library -go 1.19 +go 1.23 require gopkg.in/yaml.v3 v3.0.1 + +require k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 // indirect diff --git a/scripts/website/go.sum b/scripts/website/go.sum index a62c313c5..bcfe5c8aa 100644 --- a/scripts/website/go.sum +++ b/scripts/website/go.sum @@ -2,3 +2,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2 h1:AZYQSJemyQB5eRxqcPky+/7EdBj0xi3g0ZcxxJ7vbWU= +k8s.io/utils v0.0.0-20260210185600-b8788abfbbc2/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= diff --git a/scripts/website/sidebars-template.js b/scripts/website/sidebars-template.js index 2370e35d3..9b5ecf434 100644 --- a/scripts/website/sidebars-template.js +++ b/scripts/website/sidebars-template.js @@ -21,6 +21,52 @@ module.exports = { %GENERAL_ITEMS% ], }, + { + type: 'category', + label: 'AI Workload Policies', + collapsed: true, + items: [ + { + type: 'category', + label: 'Profiles', + collapsed: true, + items: [ + { + type: 'category', + label: 'GPU Safety', + collapsed: true, + items: [ +%AI_WORKLOAD_SAFETY_ITEMS% + ], + }, + { + type: 'category', + label: 'Training', + collapsed: true, + items: [ +%AI_WORKLOAD_TRAINING_ITEMS% + ], + }, + { + type: 'category', + label: 'Inference', + collapsed: true, + items: [ +%AI_WORKLOAD_INFERENCE_ITEMS% + ], + }, + ], + }, + { + type: 'category', + label: 'Other', + collapsed: true, + items: [ +%OTHER_AI_WORKLOAD_ITEMS% + ], + }, + ], + }, { type: 'category', label: 'Pod Security Standards', diff --git a/src/general/gpuactivedeadline/constraint.tmpl b/src/general/gpuactivedeadline/constraint.tmpl new file mode 100644 index 000000000..2dce4f58f --- /dev/null +++ b/src/general/gpuactivedeadline/constraint.tmpl @@ -0,0 +1,48 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuactivedeadline + annotations: + metadata.gatekeeper.sh/title: "GPU Active Deadline Required" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-ai-training-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set + activeDeadlineSeconds. This prevents runaway training jobs from holding + GPU resources indefinitely. +spec: + crd: + spec: + names: + kind: K8sGpuActiveDeadline + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to set activeDeadlineSeconds. + properties: + maxActiveDeadlineSeconds: + description: >- + The maximum value allowed for activeDeadlineSeconds. Set to 0 to + only require the field is present without enforcing a maximum. + type: integer + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: +{{ file.Read "src/general/gpuactivedeadline/src.cel" | strings.Indent 10 | strings.TrimSuffix "\n" }} + - engine: Rego + source: + rego: | +{{ file.Read "src/general/gpuactivedeadline/src.rego" | strings.Indent 12 | strings.TrimSuffix "\n" }} + libs: + - | +{{ file.Read "src/general/gpuactivedeadline/lib_exempt_container.rego" | strings.Indent 14 | strings.TrimSuffix "\n" }} diff --git a/src/general/gpuactivedeadline/lib_exempt_container.rego b/src/general/gpuactivedeadline/lib_exempt_container.rego new file mode 100644 index 000000000..43e84d75e --- /dev/null +++ b/src/general/gpuactivedeadline/lib_exempt_container.rego @@ -0,0 +1,19 @@ +package lib.exempt_container + +is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) +} + +_matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img +} + +_matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) +} \ No newline at end of file diff --git a/src/general/gpuactivedeadline/src.cel b/src/general/gpuactivedeadline/src.cel new file mode 100644 index 000000000..3264dfab4 --- /dev/null +++ b/src/general/gpuactivedeadline/src.cel @@ -0,0 +1,39 @@ +variables: +- name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' +- name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' +- name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' +- name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) +- name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) +- name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) +- name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) +- name: hasDeadline + expression: 'has(variables.anyObject.spec.activeDeadlineSeconds)' +- name: maxDeadline + expression: 'has(variables.params.maxActiveDeadlineSeconds) ? variables.params.maxActiveDeadlineSeconds : 0' +validations: +- expression: '!variables.podRequestsGpu || variables.hasDeadline' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not set activeDeadlineSeconds"' +- expression: '!variables.podRequestsGpu || !variables.hasDeadline || variables.maxDeadline == 0 || variables.anyObject.spec.activeDeadlineSeconds <= variables.maxDeadline' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> sets activeDeadlineSeconds to " + string(variables.anyObject.spec.activeDeadlineSeconds) + ", which exceeds the maximum allowed " + string(variables.maxDeadline)' diff --git a/src/general/gpuactivedeadline/src.rego b/src/general/gpuactivedeadline/src.rego new file mode 100644 index 000000000..1a71eff8c --- /dev/null +++ b/src/general/gpuactivedeadline/src.rego @@ -0,0 +1,42 @@ +package k8sgpuactivedeadline + +import data.lib.exempt_container.is_exempt + +violation[{"msg": msg}] { + pod_requests_gpu + not has_active_deadline + msg := sprintf("Pod <%v> requests GPU resources but does not set activeDeadlineSeconds", [input.review.object.metadata.name]) +} + +violation[{"msg": msg}] { + pod_requests_gpu + has_active_deadline + max_deadline := object.get(input, ["parameters", "maxActiveDeadlineSeconds"], 0) + max_deadline > 0 + deadline := input.review.object.spec.activeDeadlineSeconds + deadline > max_deadline + msg := sprintf("Pod <%v> sets activeDeadlineSeconds to %v, which exceeds the maximum allowed %v", [input.review.object.metadata.name, deadline, max_deadline]) +} + +pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +has_active_deadline { + input.review.object.spec.activeDeadlineSeconds +} + +input_containers[c] { + c := input.review.object.spec.containers[_] +} + +input_containers[c] { + c := input.review.object.spec.initContainers[_] +} + +input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] +} diff --git a/src/general/gpuactivedeadline/src_test.rego b/src/general/gpuactivedeadline/src_test.rego new file mode 100644 index 000000000..67360f994 --- /dev/null +++ b/src/general/gpuactivedeadline/src_test.rego @@ -0,0 +1,41 @@ +package k8sgpuactivedeadline + +test_gpu_with_deadline_allowed { + inp := {"review": review_with_deadline([gpu_container("train", "nvidia/cuda:12.0", "1")], 3600), "parameters": {"maxActiveDeadlineSeconds": 86400}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_without_deadline_denied { + inp := {"review": review([gpu_container("train", "nvidia/cuda:12.0", "1")]), "parameters": {}} + results := violation with input as inp + count(results) == 1 +} + +test_gpu_exceeds_max_deadline_denied { + inp := {"review": review_with_deadline([gpu_container("train", "nvidia/cuda:12.0", "1")], 172800), "parameters": {"maxActiveDeadlineSeconds": 86400}} + results := violation with input as inp + count(results) == 1 +} + +test_no_gpu_allowed { + inp := {"review": review([no_gpu_container("web", "nginx:latest")]), "parameters": {}} + results := violation with input as inp + count(results) == 0 +} + +review(containers) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers}}} +} + +review_with_deadline(containers, deadline) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers, "activeDeadlineSeconds": deadline}}} +} + +gpu_container(name, image, gpus) = c { + c := {"name": name, "image": image, "resources": {"limits": {"nvidia.com/gpu": gpus}}} +} + +no_gpu_container(name, image) = c { + c := {"name": name, "image": image, "resources": {"limits": {"cpu": "100m"}}} +} diff --git a/src/general/gpunodetargeting/constraint.tmpl b/src/general/gpunodetargeting/constraint.tmpl new file mode 100644 index 000000000..97db8f2af --- /dev/null +++ b/src/general/gpunodetargeting/constraint.tmpl @@ -0,0 +1,58 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpunodetargeting + annotations: + metadata.gatekeeper.sh/title: "GPU Node Targeting" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to target + GPU-labeled nodes using required node affinity or nodeSelector. This helps + ensure GPU workloads only land on nodes that advertise GPU capacity. +spec: + crd: + spec: + names: + kind: K8sGpuNodeTargeting + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to target nodes with a configured GPU label key and optional values. + properties: + nodeLabelKey: + description: >- + The node label key that GPU workloads must target (for example, `nvidia.com/gpu.present` + or `nvidia.com/gpu.product`). + type: string + nodeLabelValues: + description: >- + Optional allowed values for the GPU node label. If omitted, the policy only requires the + label key to be present. + type: array + items: + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: +{{ file.Read "src/general/gpunodetargeting/src.cel" | strings.Indent 10 | strings.TrimSuffix "\n" }} + - engine: Rego + source: + rego: | +{{ file.Read "src/general/gpunodetargeting/src.rego" | strings.Indent 12 | strings.TrimSuffix "\n" }} + libs: + - | +{{ file.Read "src/general/gpunodetargeting/lib_exempt_container.rego" | strings.Indent 14 | strings.TrimSuffix "\n" }} \ No newline at end of file diff --git a/src/general/gpunodetargeting/lib_exempt_container.rego b/src/general/gpunodetargeting/lib_exempt_container.rego new file mode 100644 index 000000000..43e84d75e --- /dev/null +++ b/src/general/gpunodetargeting/lib_exempt_container.rego @@ -0,0 +1,19 @@ +package lib.exempt_container + +is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) +} + +_matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img +} + +_matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) +} \ No newline at end of file diff --git a/src/general/gpunodetargeting/src.cel b/src/general/gpunodetargeting/src.cel new file mode 100644 index 000000000..7185f34a1 --- /dev/null +++ b/src/general/gpunodetargeting/src.cel @@ -0,0 +1,73 @@ +variables: +- name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' +- name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' +- name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' +- name: allContainers + expression: 'variables.containers + variables.initContainers + variables.ephemeralContainers' +- name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) +- name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) +- name: exemptImages + expression: | + variables.allContainers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) +- name: podRequestsGpu + expression: | + variables.allContainers.exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + ( + (has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) || + (has(container.resources.requests) && + "nvidia.com/gpu" in container.resources.requests && + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) + ) + ) +- name: nodeLabelKey + expression: 'has(variables.params.nodeLabelKey) ? variables.params.nodeLabelKey : ""' +- name: nodeLabelValues + expression: 'has(variables.params.nodeLabelValues) ? variables.params.nodeLabelValues : []' +- name: hasMatchingNodeSelector + expression: | + !has(variables.anyObject.spec.nodeSelector) || !(variables.nodeLabelKey in variables.anyObject.spec.nodeSelector) ? false : + (size(variables.nodeLabelValues) == 0 ? + string(variables.anyObject.spec.nodeSelector[variables.nodeLabelKey]) != "" : + variables.anyObject.spec.nodeSelector[variables.nodeLabelKey] in variables.nodeLabelValues) +- name: hasMatchingNodeAffinity + expression: | + !has(variables.anyObject.spec.affinity) || + !has(variables.anyObject.spec.affinity.nodeAffinity) || + !has(variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution) || + !has(variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms) ? false : + variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms.exists(term, + has(term.matchExpressions) && + term.matchExpressions.exists(expr, + expr.key == variables.nodeLabelKey && + ( + size(variables.nodeLabelValues) == 0 ? + expr.operator == "Exists" : + expr.operator == "In" && + has(expr.values) && + size(expr.values) > 0 && + expr.values.all(exprValue, exprValue in variables.nodeLabelValues) + ) + ) + ) +validations: +- expression: '!variables.podRequestsGpu || variables.nodeLabelKey == "" || variables.hasMatchingNodeSelector || variables.hasMatchingNodeAffinity' + messageExpression: | + size(variables.nodeLabelValues) == 0 ? + "Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not target nodes with label key <" + variables.nodeLabelKey + "> using node affinity or nodeSelector" : + "Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not target nodes with label <" + variables.nodeLabelKey + "> matching one of <" + variables.nodeLabelValues.join(", ") + "> using node affinity or nodeSelector" \ No newline at end of file diff --git a/src/general/gpunodetargeting/src.rego b/src/general/gpunodetargeting/src.rego new file mode 100644 index 000000000..3619c4389 --- /dev/null +++ b/src/general/gpunodetargeting/src.rego @@ -0,0 +1,98 @@ +package k8sgpunodetargeting + +import data.lib.exempt_container.is_exempt + +violation[{"msg": msg}] { + pod_requests_gpu + label_key := object.get(input.parameters, "nodeLabelKey", "") + label_key != "" + not has_matching_node_selector(label_key) + not has_matching_node_affinity(label_key) + label_values := object.get(input.parameters, "nodeLabelValues", []) + msg := violation_message(label_key, label_values) +} + +violation_message(label_key, label_values) = msg { + count(label_values) == 0 + msg := sprintf("Pod <%v> requests GPU resources but does not target nodes with label key <%v> using node affinity or nodeSelector", [input.review.object.metadata.name, label_key]) +} + +violation_message(label_key, label_values) = msg { + count(label_values) > 0 + msg := sprintf("Pod <%v> requests GPU resources but does not target nodes with label <%v> matching one of <%v> using node affinity or nodeSelector", [input.review.object.metadata.name, label_key, label_values]) +} + +pod_requests_gpu { + container := all_containers[_] + not is_exempt(container) + requests_gpu(container) +} + +all_containers[c] { + c := input.review.object.spec.containers[_] +} + +all_containers[c] { + c := input.review.object.spec.initContainers[_] +} + +all_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] +} + +requests_gpu(container) { + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu := limits["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +requests_gpu(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + gpu := requests["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +has_matching_node_selector(label_key) { + selector := input.review.object.spec.nodeSelector + value := selector[label_key] + value != "" + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) == 0 +} + +has_matching_node_selector(label_key) { + selector := input.review.object.spec.nodeSelector + value := selector[label_key] + label_values := object.get(input.parameters, "nodeLabelValues", []) + label_values[_] == value +} + +has_matching_node_affinity(label_key) { + term := input.review.object.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[_] + expr := term.matchExpressions[_] + expr.key == label_key + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) == 0 + expr.operator == "Exists" +} + +has_matching_node_affinity(label_key) { + term := input.review.object.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[_] + expr := term.matchExpressions[_] + expr.key == label_key + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) > 0 + expr.operator == "In" + values := object.get(expr, "values", []) + count(values) > 0 + not has_disallowed_affinity_value(values, label_values) +} + +has_disallowed_affinity_value(values, label_values) { + value := values[_] + not allowed_affinity_value(value, label_values) +} + +allowed_affinity_value(value, label_values) { + label_values[_] == value +} diff --git a/src/general/gpunodetargeting/src_test.rego b/src/general/gpunodetargeting/src_test.rego new file mode 100644 index 000000000..e0166db6b --- /dev/null +++ b/src/general/gpunodetargeting/src_test.rego @@ -0,0 +1,91 @@ +package k8sgpunodetargeting + +test_gpu_pod_with_node_affinity_allowed { + inp := {"review": review_with_affinity([gpu_container("trainer")], required_gpu_affinity("nvidia.com/gpu.present", ["true"])), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present", "nodeLabelValues": ["true"]}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_pod_with_node_selector_allowed { + inp := {"review": review_with_node_selector([gpu_container("trainer")], {"nvidia.com/gpu.present": "true"}), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present", "nodeLabelValues": ["true"]}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_pod_node_selector_key_only_allowed { + inp := {"review": review_with_node_selector([gpu_container("trainer")], {"nvidia.com/gpu.present": "true"}), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present"}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_pod_node_selector_key_only_empty_value_denied { + inp := {"review": review_with_node_selector([gpu_container("trainer")], {"nvidia.com/gpu.present": ""}), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present"}} + results := violation with input as inp + count(results) == 1 +} + +test_gpu_pod_without_targeting_denied { + inp := {"review": review([gpu_container("trainer")]), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present", "nodeLabelValues": ["true"]}} + results := violation with input as inp + count(results) == 1 +} + +test_gpu_pod_wrong_node_label_value_denied { + inp := {"review": review_with_affinity([gpu_container("trainer")], required_gpu_affinity("nvidia.com/gpu.present", ["false"])), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present", "nodeLabelValues": ["true"]}} + results := violation with input as inp + count(results) == 1 +} + +test_gpu_pod_mixed_node_label_values_denied { + inp := {"review": review_with_affinity([gpu_container("trainer")], required_gpu_affinity("nvidia.com/gpu.present", ["true", "false"])), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present", "nodeLabelValues": ["true"]}} + results := violation with input as inp + count(results) == 1 +} + +test_non_gpu_pod_allowed { + inp := {"review": review([non_gpu_container("web")]), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present", "nodeLabelValues": ["true"]}} + results := violation with input as inp + count(results) == 0 +} + +test_exempt_gpu_container_allowed { + inp := {"review": review([gpu_container_with_image("monitor", "nvidia/dcgm-exporter:3.1")]), "parameters": {"nodeLabelKey": "nvidia.com/gpu.present", "nodeLabelValues": ["true"], "exemptImages": ["nvidia/dcgm-exporter:*"]}} + results := violation with input as inp + count(results) == 0 +} + +review(containers) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers}}} +} + +review_with_affinity(containers, affinity) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers, "affinity": affinity}}} +} + +review_with_node_selector(containers, node_selector) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers, "nodeSelector": node_selector}}} +} + +required_gpu_affinity(key, values) = affinity { + affinity := { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + {"matchExpressions": [{"key": key, "operator": "In", "values": values}]}, + ], + }, + }, + } +} + +gpu_container(name) = c { + c := gpu_container_with_image(name, "nvidia/cuda:12.0-runtime") +} + +gpu_container_with_image(name, image) = c { + c := {"name": name, "image": image, "resources": {"limits": {"nvidia.com/gpu": "1"}}} +} + +non_gpu_container(name) = c { + c := {"name": name, "image": "nginx:1.25", "resources": {"limits": {"cpu": "500m", "memory": "128Mi"}}} +} \ No newline at end of file diff --git a/src/general/gpuresourcelimits/constraint.tmpl b/src/general/gpuresourcelimits/constraint.tmpl new file mode 100644 index 000000000..c55e330ea --- /dev/null +++ b/src/general/gpuresourcelimits/constraint.tmpl @@ -0,0 +1,47 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuresourcelimits + annotations: + metadata.gatekeeper.sh/title: "GPU Resource Limits" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Enforces a maximum number of NVIDIA GPUs (nvidia.com/gpu) that a single + container may request. This prevents individual containers from hoarding + GPU resources on shared clusters, particularly for AI/ML training workloads. +spec: + crd: + spec: + names: + kind: K8sGpuResourceLimits + validation: + openAPIV3Schema: + type: object + description: >- + Enforces a maximum number of NVIDIA GPUs per container. + properties: + maxGpuPerContainer: + description: >- + The maximum number of GPUs a single container may request. + type: integer + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: +{{ file.Read "src/general/gpuresourcelimits/src.cel" | strings.Indent 10 | strings.TrimSuffix "\n" }} + - engine: Rego + source: + rego: | +{{ file.Read "src/general/gpuresourcelimits/src.rego" | strings.Indent 12 | strings.TrimSuffix "\n" }} + libs: + - | +{{ file.Read "src/general/gpuresourcelimits/lib_exempt_container.rego" | strings.Indent 14 | strings.TrimSuffix "\n" }} diff --git a/src/general/gpuresourcelimits/lib_exempt_container.rego b/src/general/gpuresourcelimits/lib_exempt_container.rego new file mode 100644 index 000000000..43e84d75e --- /dev/null +++ b/src/general/gpuresourcelimits/lib_exempt_container.rego @@ -0,0 +1,19 @@ +package lib.exempt_container + +is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) +} + +_matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img +} + +_matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) +} \ No newline at end of file diff --git a/src/general/gpuresourcelimits/src.cel b/src/general/gpuresourcelimits/src.cel new file mode 100644 index 000000000..a8b7204d6 --- /dev/null +++ b/src/general/gpuresourcelimits/src.cel @@ -0,0 +1,37 @@ +variables: +- name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' +- name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' +- name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' +- name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) +- name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) +- name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) +- name: maxGpu + expression: 'has(variables.params.maxGpuPerContainer) ? variables.params.maxGpuPerContainer : 0' +- name: badContainers + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + variables.maxGpu > 0 && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity(string(variables.maxGpu))) > 0 + ).map(container, "Container <" + container.name + "> requests " + string(container.resources.limits["nvidia.com/gpu"]) + " GPUs, which exceeds the maximum allowed " + string(variables.maxGpu)) +validations: +- expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' diff --git a/src/general/gpuresourcelimits/src.rego b/src/general/gpuresourcelimits/src.rego new file mode 100644 index 000000000..6d6bb9e67 --- /dev/null +++ b/src/general/gpuresourcelimits/src.rego @@ -0,0 +1,26 @@ +package k8sgpuresourcelimits + +import data.lib.exempt_container.is_exempt + +violation[{"msg": msg}] { + container := input_containers[_] + not is_exempt(container) + gpu_count := to_number(container.resources.limits["nvidia.com/gpu"]) + gpu_count > 0 + max_gpu := object.get(input, ["parameters", "maxGpuPerContainer"], 0) + max_gpu > 0 + gpu_count > max_gpu + msg := sprintf("Container <%v> requests %v GPUs, which exceeds the maximum allowed %v", [container.name, gpu_count, max_gpu]) +} + +input_containers[c] { + c := input.review.object.spec.containers[_] +} + +input_containers[c] { + c := input.review.object.spec.initContainers[_] +} + +input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] +} diff --git a/src/general/gpuresourcelimits/src_test.rego b/src/general/gpuresourcelimits/src_test.rego new file mode 100644 index 000000000..9afca1cc9 --- /dev/null +++ b/src/general/gpuresourcelimits/src_test.rego @@ -0,0 +1,49 @@ +package k8sgpuresourcelimits + +test_gpu_within_limit_allowed { + inp := {"review": review([gpu_container("training", "nvidia/cuda:12.0", "2")]), "parameters": {"maxGpuPerContainer": 4}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_exceeds_limit_denied { + inp := {"review": review([gpu_container("training", "nvidia/cuda:12.0", "8")]), "parameters": {"maxGpuPerContainer": 4}} + results := violation with input as inp + count(results) == 1 +} + +test_no_gpu_allowed { + inp := {"review": review([no_gpu_container("web", "nginx:latest")]), "parameters": {"maxGpuPerContainer": 4}} + results := violation with input as inp + count(results) == 0 +} + +test_exempt_image_allowed { + inp := {"review": review([gpu_container("monitor", "nvidia/dcgm-exporter:3.1", "8")]), "parameters": {"maxGpuPerContainer": 4, "exemptImages": ["nvidia/dcgm-exporter:*"]}} + results := violation with input as inp + count(results) == 0 +} + +test_multiple_containers_mixed { + inp := {"review": review([gpu_container("ok", "nvidia/cuda:12.0", "2"), gpu_container("bad", "myrepo/train:v1", "8")]), "parameters": {"maxGpuPerContainer": 4}} + results := violation with input as inp + count(results) == 1 +} + +test_zero_gpu_allowed { + inp := {"review": review([gpu_container("idle", "myrepo/train:v1", "0")]), "parameters": {"maxGpuPerContainer": 4}} + results := violation with input as inp + count(results) == 0 +} + +review(containers) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers}}} +} + +gpu_container(name, image, gpus) = c { + c := {"name": name, "image": image, "resources": {"limits": {"nvidia.com/gpu": gpus}}} +} + +no_gpu_container(name, image) = c { + c := {"name": name, "image": image, "resources": {"limits": {"cpu": "100m", "memory": "128Mi"}}} +} diff --git a/src/general/gpusharedmemory/constraint.tmpl b/src/general/gpusharedmemory/constraint.tmpl new file mode 100644 index 000000000..d1665db5f --- /dev/null +++ b/src/general/gpusharedmemory/constraint.tmpl @@ -0,0 +1,44 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpusharedmemory + annotations: + metadata.gatekeeper.sh/title: "GPU Shared Memory Required" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-ai-training-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to mount a + memory-backed emptyDir volume at /dev/shm. PyTorch DataLoader, NCCL + multi-GPU communication, and most training frameworks require shared memory + beyond the default 64MB. +spec: + crd: + spec: + names: + kind: K8sGpuSharedMemory + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to mount a memory-backed volume at /dev/shm. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: +{{ file.Read "src/general/gpusharedmemory/src.cel" | strings.Indent 10 | strings.TrimSuffix "\n" }} + - engine: Rego + source: + rego: | +{{ file.Read "src/general/gpusharedmemory/src.rego" | strings.Indent 12 | strings.TrimSuffix "\n" }} + libs: + - | +{{ file.Read "src/general/gpusharedmemory/lib_exempt_container.rego" | strings.Indent 14 | strings.TrimSuffix "\n" }} diff --git a/src/general/gpusharedmemory/lib_exempt_container.rego b/src/general/gpusharedmemory/lib_exempt_container.rego new file mode 100644 index 000000000..43e84d75e --- /dev/null +++ b/src/general/gpusharedmemory/lib_exempt_container.rego @@ -0,0 +1,19 @@ +package lib.exempt_container + +is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) +} + +_matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img +} + +_matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) +} \ No newline at end of file diff --git a/src/general/gpusharedmemory/src.cel b/src/general/gpusharedmemory/src.cel new file mode 100644 index 000000000..fadbecc3d --- /dev/null +++ b/src/general/gpusharedmemory/src.cel @@ -0,0 +1,42 @@ +variables: +- name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' +- name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) +- name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) +- name: exemptImages + expression: | + variables.containers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) +- name: volumes + expression: 'has(variables.anyObject.spec.volumes) ? variables.anyObject.spec.volumes : []' +- name: memoryVolNames + expression: | + variables.volumes.filter(v, + has(v.emptyDir) && has(v.emptyDir.medium) && v.emptyDir.medium == "Memory" + ).map(v, v.name) +- name: badContainers + expression: | + variables.containers.filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + (!has(container.volumeMounts) || + !container.volumeMounts.exists(vm, + vm.mountPath == "/dev/shm" && + vm.name in variables.memoryVolNames + ) + ) + ).map(container, "Container <" + container.name + "> requests GPU resources but does not mount a memory-backed volume at /dev/shm") +validations: +- expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' diff --git a/src/general/gpusharedmemory/src.rego b/src/general/gpusharedmemory/src.rego new file mode 100644 index 000000000..2b05d0268 --- /dev/null +++ b/src/general/gpusharedmemory/src.rego @@ -0,0 +1,24 @@ +package k8sgpusharedmemory + +import data.lib.exempt_container.is_exempt + +violation[{"msg": msg}] { + container := input.review.object.spec.containers[_] + not is_exempt(container) + has_gpu_request(container) + not has_shm_mount(container) + msg := sprintf("Container <%v> requests GPU resources but does not mount a memory-backed volume at /dev/shm", [container.name]) +} + +has_gpu_request(container) { + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +has_shm_mount(container) { + mount := container.volumeMounts[_] + mount.mountPath == "/dev/shm" + volume := input.review.object.spec.volumes[_] + volume.name == mount.name + volume.emptyDir.medium == "Memory" +} diff --git a/src/general/gpusharedmemory/src_test.rego b/src/general/gpusharedmemory/src_test.rego new file mode 100644 index 000000000..551614bf7 --- /dev/null +++ b/src/general/gpusharedmemory/src_test.rego @@ -0,0 +1,39 @@ +package k8sgpusharedmemory + +test_gpu_with_shm_allowed { + inp := {"review": review_with_shm([gpu_container_with_shm("train", "nvidia/cuda:12.0", "1")]), "parameters": {}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_without_shm_denied { + inp := {"review": review([gpu_container("train", "nvidia/cuda:12.0", "1")]), "parameters": {}} + results := violation with input as inp + count(results) == 1 +} + +test_no_gpu_allowed { + inp := {"review": review([no_gpu_container("web", "nginx:latest")]), "parameters": {}} + results := violation with input as inp + count(results) == 0 +} + +review(containers) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers}}} +} + +review_with_shm(containers) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers, "volumes": [{"name": "dshm", "emptyDir": {"medium": "Memory"}}]}}} +} + +gpu_container(name, image, gpus) = c { + c := {"name": name, "image": image, "resources": {"limits": {"nvidia.com/gpu": gpus}}} +} + +gpu_container_with_shm(name, image, gpus) = c { + c := {"name": name, "image": image, "resources": {"limits": {"nvidia.com/gpu": gpus}}, "volumeMounts": [{"name": "dshm", "mountPath": "/dev/shm"}]} +} + +no_gpu_container(name, image) = c { + c := {"name": name, "image": image, "resources": {"limits": {"cpu": "100m"}}} +} diff --git a/src/general/gpuworkloadresources/constraint.tmpl b/src/general/gpuworkloadresources/constraint.tmpl new file mode 100644 index 000000000..c7cbaa059 --- /dev/null +++ b/src/general/gpuworkloadresources/constraint.tmpl @@ -0,0 +1,49 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuworkloadresources + annotations: + metadata.gatekeeper.sh/title: "GPU Workload Resources" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set + memory requests equal to limits for all non-exempt containers and to set + a CPU request. Containers that request GPUs must also set matching GPU + requests and limits. This keeps GPU workloads schedulable and predictable + while still allowing CPU burst above request. +spec: + crd: + spec: + names: + kind: K8sGpuWorkloadResources + validation: + openAPIV3Schema: + type: object + description: >- + Enforces memory request/limit alignment and cpu requests for GPU pods, + plus matching gpu requests and limits on GPU containers. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: +{{ file.Read "src/general/gpuworkloadresources/src.cel" | strings.Indent 10 | strings.TrimSuffix "\n" }} + - engine: Rego + source: + rego: | +{{ file.Read "src/general/gpuworkloadresources/src.rego" | strings.Indent 12 | strings.TrimSuffix "\n" }} + libs: + - | +{{ file.Read "src/general/gpuworkloadresources/lib_exempt_container.rego" | strings.Indent 14 | strings.TrimSuffix "\n" }} \ No newline at end of file diff --git a/src/general/gpuworkloadresources/lib_exempt_container.rego b/src/general/gpuworkloadresources/lib_exempt_container.rego new file mode 100644 index 000000000..43e84d75e --- /dev/null +++ b/src/general/gpuworkloadresources/lib_exempt_container.rego @@ -0,0 +1,19 @@ +package lib.exempt_container + +is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) +} + +_matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img +} + +_matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) +} \ No newline at end of file diff --git a/src/general/gpuworkloadresources/src.cel b/src/general/gpuworkloadresources/src.cel new file mode 100644 index 000000000..4a8b73ad7 --- /dev/null +++ b/src/general/gpuworkloadresources/src.cel @@ -0,0 +1,85 @@ +variables: +- name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' +- name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' +- name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' +- name: enforcedContainers + expression: 'variables.containers + variables.initContainers' +- name: allContainers + expression: 'variables.enforcedContainers + variables.ephemeralContainers' +- name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) +- name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) +- name: exemptImages + expression: | + variables.allContainers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) +- name: gpuContainers + expression: | + variables.allContainers.filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + ( + (has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) || + (has(container.resources.requests) && + "nvidia.com/gpu" in container.resources.requests && + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) + ) + ) +- name: podRequestsGpu + expression: 'size(variables.gpuContainers) > 0' +- name: gpuRequestLimitMessages + expression: | + variables.gpuContainers.filter(container, + !has(container.resources.requests) || + !("nvidia.com/gpu" in container.resources.requests) || + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) <= 0 || + !has(container.resources.limits) || + !("nvidia.com/gpu" in container.resources.limits) || + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) <= 0 || + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity(string(container.resources.limits["nvidia.com/gpu"]))) != 0 + ).map(container, "Container <" + container.name + "> must set nvidia.com/gpu request equal to limit") +- name: memoryRequestLimitMessages + expression: | + !variables.podRequestsGpu ? [] : + variables.enforcedContainers.filter(container, + !(container.image in variables.exemptImages) && + ( + !has(container.resources) || + !has(container.resources.requests) || + !("memory" in container.resources.requests) || + !has(container.resources.limits) || + !("memory" in container.resources.limits) || + quantity(string(container.resources.requests["memory"])).compareTo(quantity(string(container.resources.limits["memory"]))) != 0 + ) + ).map(container, "Container <" + container.name + "> in a GPU pod must set memory request equal to limit") +- name: cpuRequestMessages + expression: | + !variables.podRequestsGpu ? [] : + variables.enforcedContainers.filter(container, + !(container.image in variables.exemptImages) && + ( + !has(container.resources) || + !has(container.resources.requests) || + !("cpu" in container.resources.requests) || + string(container.resources.requests["cpu"]) == "" + ) + ).map(container, "Container <" + container.name + "> in a GPU pod must set a cpu request") +validations: +- expression: 'size(variables.gpuRequestLimitMessages) == 0' + messageExpression: 'variables.gpuRequestLimitMessages.join(", ")' +- expression: 'size(variables.memoryRequestLimitMessages) == 0' + messageExpression: 'variables.memoryRequestLimitMessages.join(", ")' +- expression: 'size(variables.cpuRequestMessages) == 0' + messageExpression: 'variables.cpuRequestMessages.join(", ")' \ No newline at end of file diff --git a/src/general/gpuworkloadresources/src.rego b/src/general/gpuworkloadresources/src.rego new file mode 100644 index 000000000..b033bf148 --- /dev/null +++ b/src/general/gpuworkloadresources/src.rego @@ -0,0 +1,191 @@ +package k8sgpuworkloadresources + +import data.lib.exempt_container.is_exempt + +missing(obj, field) = true { + not obj[field] +} + +missing(obj, field) = true { + obj[field] == "" +} + +# 10 ** 21 +mem_multiple("E") = 1000000000000000000000 { true } + +# 10 ** 18 +mem_multiple("P") = 1000000000000000000 { true } + +# 10 ** 15 +mem_multiple("T") = 1000000000000000 { true } + +# 10 ** 12 +mem_multiple("G") = 1000000000000 { true } + +# 10 ** 9 +mem_multiple("M") = 1000000000 { true } + +# 10 ** 6 +mem_multiple("k") = 1000000 { true } + +# 10 ** 3 +mem_multiple("") = 1000 { true } + +# Kubernetes accepts millibyte precision when it probably shouldn't. +# https://github.com/kubernetes/kubernetes/issues/28741 +# 10 ** 0 +mem_multiple("m") = 1 { true } + +# 1000 * 2 ** 10 +mem_multiple("Ki") = 1024000 { true } + +# 1000 * 2 ** 20 +mem_multiple("Mi") = 1048576000 { true } + +# 1000 * 2 ** 30 +mem_multiple("Gi") = 1073741824000 { true } + +# 1000 * 2 ** 40 +mem_multiple("Ti") = 1099511627776000 { true } + +# 1000 * 2 ** 50 +mem_multiple("Pi") = 1125899906842624000 { true } + +# 1000 * 2 ** 60 +mem_multiple("Ei") = 1152921504606846976000 { true } + +get_suffix(mem) = suffix { + not is_string(mem) + suffix := "" +} + +get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 0 + suffix := substring(mem, count(mem) - 1, -1) + mem_multiple(suffix) +} + +get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 1 + suffix := substring(mem, count(mem) - 2, -1) + mem_multiple(suffix) +} + +get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 1 + not mem_multiple(substring(mem, count(mem) - 1, -1)) + not mem_multiple(substring(mem, count(mem) - 2, -1)) + suffix := "" +} + +get_suffix(mem) = suffix { + is_string(mem) + count(mem) == 1 + not mem_multiple(substring(mem, count(mem) - 1, -1)) + suffix := "" +} + +get_suffix(mem) = suffix { + is_string(mem) + count(mem) == 0 + suffix := "" +} + +canonify_mem(orig) = new { + is_number(orig) + new := orig * 1000 +} + +canonify_mem(orig) = new { + not is_number(orig) + suffix := get_suffix(orig) + raw := replace(orig, suffix, "") + regex.match("^[0-9]+(\\.[0-9]+)?$", raw) + new := to_number(raw) * mem_multiple(suffix) +} + +violation[{"msg": msg}] { + container := gpu_containers[_] + not has_matching_gpu_request_and_limit(container) + msg := sprintf("Container <%v> must set nvidia.com/gpu request equal to limit", [container.name]) +} + +violation[{"msg": msg}] { + pod_requests_gpu + container := enforced_containers[_] + not is_exempt(container) + not has_matching_memory_request_and_limit(container) + msg := sprintf("Container <%v> in a GPU pod must set memory request equal to limit", [container.name]) +} + +violation[{"msg": msg}] { + pod_requests_gpu + container := enforced_containers[_] + not is_exempt(container) + not has_cpu_request(container) + msg := sprintf("Container <%v> in a GPU pod must set a cpu request", [container.name]) +} + +pod_requests_gpu { + gpu_containers[_] +} + +gpu_containers[c] { + c := all_containers[_] + not is_exempt(c) + requests_gpu(c) +} + +enforced_containers[c] { + c := input.review.object.spec.containers[_] +} + +enforced_containers[c] { + c := input.review.object.spec.initContainers[_] +} + +all_containers[c] { + c := enforced_containers[_] +} + +all_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] +} + +requests_gpu(container) { + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu := limits["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +requests_gpu(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + gpu := requests["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +has_matching_gpu_request_and_limit(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu_request := requests["nvidia.com/gpu"] + gpu_limit := limits["nvidia.com/gpu"] + to_number(gpu_request) > 0 + to_number(gpu_limit) > 0 + to_number(gpu_request) == to_number(gpu_limit) +} + +has_matching_memory_request_and_limit(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + limits := object.get(object.get(container, "resources", {}), "limits", {}) + mem_request := requests["memory"] + mem_limit := limits["memory"] + canonify_mem(mem_request) == canonify_mem(mem_limit) +} + +has_cpu_request(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + not missing(requests, "cpu") +} \ No newline at end of file diff --git a/src/general/gpuworkloadresources/src_test.rego b/src/general/gpuworkloadresources/src_test.rego new file mode 100644 index 000000000..552b70c06 --- /dev/null +++ b/src/general/gpuworkloadresources/src_test.rego @@ -0,0 +1,111 @@ +package k8sgpuworkloadresources + +test_gpu_pod_compliant_allowed { + inp := {"review": review([compliant_gpu_container("trainer"), compliant_sidecar("logger")]), "parameters": {}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_container_missing_gpu_request_denied { + inp := {"review": review([gpu_limit_only_container("trainer")]), "parameters": {}} + results := violation with input as inp + count(results) == 1 +} + +test_gpu_pod_memory_mismatch_denied { + inp := {"review": review([gpu_memory_mismatch_container("trainer")]), "parameters": {}} + results := violation with input as inp + count(results) == 1 +} + +test_gpu_pod_sidecar_missing_cpu_request_denied { + inp := {"review": review([compliant_gpu_container("trainer"), sidecar_missing_cpu_request("logger")]), "parameters": {}} + results := violation with input as inp + count(results) == 1 +} + +test_non_gpu_pod_allowed { + inp := {"review": review([non_gpu_container("web")]), "parameters": {}} + results := violation with input as inp + count(results) == 0 +} + +test_exempt_gpu_container_allowed { + inp := {"review": review([gpu_limit_only_container_with_image("monitor", "nvidia/dcgm-exporter:3.1")]), "parameters": {"exemptImages": ["nvidia/dcgm-exporter:*"]}} + results := violation with input as inp + count(results) == 0 +} + +review(containers) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers}}} +} + +compliant_gpu_container(name) = c { + c := { + "name": name, + "image": "nvidia/cuda:12.0-runtime", + "resources": { + "requests": {"nvidia.com/gpu": "1", "memory": "16Gi", "cpu": "2"}, + "limits": {"nvidia.com/gpu": "1", "memory": "16Gi", "cpu": "4"}, + }, + } +} + +compliant_sidecar(name) = c { + c := { + "name": name, + "image": "busybox:1.36", + "resources": { + "requests": {"memory": "256Mi", "cpu": "100m"}, + "limits": {"memory": "256Mi", "cpu": "500m"}, + }, + } +} + +gpu_limit_only_container(name) = c { + c := gpu_limit_only_container_with_image(name, "nvidia/cuda:12.0-runtime") +} + +gpu_limit_only_container_with_image(name, image) = c { + c := { + "name": name, + "image": image, + "resources": { + "requests": {"memory": "16Gi", "cpu": "2"}, + "limits": {"nvidia.com/gpu": "1", "memory": "16Gi", "cpu": "4"}, + }, + } +} + +gpu_memory_mismatch_container(name) = c { + c := { + "name": name, + "image": "nvidia/cuda:12.0-runtime", + "resources": { + "requests": {"nvidia.com/gpu": "1", "memory": "8Gi", "cpu": "2"}, + "limits": {"nvidia.com/gpu": "1", "memory": "16Gi", "cpu": "4"}, + }, + } +} + +sidecar_missing_cpu_request(name) = c { + c := { + "name": name, + "image": "busybox:1.36", + "resources": { + "requests": {"memory": "256Mi"}, + "limits": {"memory": "256Mi", "cpu": "500m"}, + }, + } +} + +non_gpu_container(name) = c { + c := { + "name": name, + "image": "nginx:1.25", + "resources": { + "requests": {"memory": "128Mi", "cpu": "100m"}, + "limits": {"memory": "256Mi", "cpu": "500m"}, + }, + } +} \ No newline at end of file diff --git a/src/general/nounsupportedgpu/constraint.tmpl b/src/general/nounsupportedgpu/constraint.tmpl new file mode 100644 index 000000000..4a0b68ae6 --- /dev/null +++ b/src/general/nounsupportedgpu/constraint.tmpl @@ -0,0 +1,47 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8snounsupportedgpu + annotations: + metadata.gatekeeper.sh/title: "No Unsupported GPU Requests" + metadata.gatekeeper.sh/version: 1.0.0 + description: >- + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set + the NVIDIA_VISIBLE_DEVICES environment variable, indicating the container + image is built to consume GPUs via the NVIDIA CUDA runtime. This prevents + GPU resource waste from containers that request GPUs but cannot use them. +spec: + crd: + spec: + names: + kind: K8sNoUnsupportedGpu + validation: + openAPIV3Schema: + type: object + description: >- + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set + the NVIDIA_VISIBLE_DEVICES environment variable. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: +{{ file.Read "src/general/nounsupportedgpu/src.cel" | strings.Indent 10 | strings.TrimSuffix "\n" }} + - engine: Rego + source: + rego: | +{{ file.Read "src/general/nounsupportedgpu/src.rego" | strings.Indent 12 | strings.TrimSuffix "\n" }} + libs: + - | +{{ file.Read "src/general/nounsupportedgpu/lib_exempt_container.rego" | strings.Indent 14 | strings.TrimSuffix "\n" }} diff --git a/src/general/nounsupportedgpu/lib_exempt_container.rego b/src/general/nounsupportedgpu/lib_exempt_container.rego new file mode 100644 index 000000000..43e84d75e --- /dev/null +++ b/src/general/nounsupportedgpu/lib_exempt_container.rego @@ -0,0 +1,19 @@ +package lib.exempt_container + +is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) +} + +_matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img +} + +_matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) +} \ No newline at end of file diff --git a/src/general/nounsupportedgpu/src.cel b/src/general/nounsupportedgpu/src.cel new file mode 100644 index 000000000..233b29a41 --- /dev/null +++ b/src/general/nounsupportedgpu/src.cel @@ -0,0 +1,34 @@ +variables: +- name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' +- name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' +- name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' +- name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) +- name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) +- name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) +- name: badContainers + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + (!has(container.env) || !container.env.exists(e, e.name == "NVIDIA_VISIBLE_DEVICES")) + ).map(container, "Container <" + container.name + "> requests nvidia.com/gpu but does not set the NVIDIA_VISIBLE_DEVICES environment variable") +validations: +- expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' diff --git a/src/general/nounsupportedgpu/src.rego b/src/general/nounsupportedgpu/src.rego new file mode 100644 index 000000000..5c7529510 --- /dev/null +++ b/src/general/nounsupportedgpu/src.rego @@ -0,0 +1,33 @@ +package k8snounsupportedgpu + +import data.lib.exempt_container.is_exempt + +violation[{"msg": msg}] { + container := input_containers[_] + not is_exempt(container) + has_gpu_request(container) + not has_nvidia_env(container) + msg := sprintf("Container <%v> requests nvidia.com/gpu but does not set the NVIDIA_VISIBLE_DEVICES environment variable", [container.name]) +} + +has_gpu_request(container) { + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +has_nvidia_env(container) { + env := container.env[_] + env.name == "NVIDIA_VISIBLE_DEVICES" +} + +input_containers[c] { + c := input.review.object.spec.containers[_] +} + +input_containers[c] { + c := input.review.object.spec.initContainers[_] +} + +input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] +} diff --git a/src/general/nounsupportedgpu/src_test.rego b/src/general/nounsupportedgpu/src_test.rego new file mode 100644 index 000000000..295fa7b39 --- /dev/null +++ b/src/general/nounsupportedgpu/src_test.rego @@ -0,0 +1,163 @@ +package k8snounsupportedgpu + +test_gpu_with_env_var_allowed { + inp := { + "review": review([gpu_container_with_env("training", "nvidia/cuda:12.0", "1")]), + "parameters": {}, + } + results := violation with input as inp + count(results) == 0 +} + +test_no_gpu_requested_allowed { + inp := { + "review": review([no_gpu_container("web", "nginx:latest")]), + "parameters": {}, + } + results := violation with input as inp + count(results) == 0 +} + +test_gpu_without_env_var_denied { + inp := { + "review": review([gpu_container_no_env("training", "myrepo/myimage:v1", "1")]), + "parameters": {}, + } + results := violation with input as inp + count(results) == 1 +} + +test_gpu_without_env_var_multiple_denied { + inp := { + "review": review([ + gpu_container_no_env("training1", "myrepo/myimage:v1", "1"), + gpu_container_no_env("training2", "myrepo/myimage:v2", "2"), + ]), + "parameters": {}, + } + results := violation with input as inp + count(results) == 2 +} + +test_gpu_zero_allowed { + inp := { + "review": review([gpu_container_no_env_zero("training", "myrepo/myimage:v1")]), + "parameters": {}, + } + results := violation with input as inp + count(results) == 0 +} + +test_mixed_containers { + inp := { + "review": review([ + gpu_container_with_env("good", "nvidia/cuda:12.0", "1"), + gpu_container_no_env("bad", "myrepo/myimage:v1", "1"), + no_gpu_container("web", "nginx:latest"), + ]), + "parameters": {}, + } + results := violation with input as inp + count(results) == 1 +} + +test_exempt_image { + inp := { + "review": review([gpu_container_no_env("training", "exempt-registry/myimage:v1", "1")]), + "parameters": {"exemptImages": ["exempt-registry/*"]}, + } + results := violation with input as inp + count(results) == 0 +} + +test_init_container_gpu_denied { + inp := { + "review": review_with_init( + [no_gpu_container("web", "nginx:latest")], + [gpu_container_no_env("init-gpu", "myrepo/init:v1", "1")], + ), + "parameters": {}, + } + results := violation with input as inp + count(results) == 1 +} + +# Helper functions +review(containers) = output { + output = { + "object": { + "metadata": { + "name": "test-pod", + }, + "spec": { + "containers": containers, + }, + }, + } +} + +review_with_init(containers, init_containers) = output { + output = { + "object": { + "metadata": { + "name": "test-pod", + }, + "spec": { + "containers": containers, + "initContainers": init_containers, + }, + }, + } +} + +gpu_container_with_env(name, image, gpus) = c { + c := { + "name": name, + "image": image, + "resources": { + "limits": { + "nvidia.com/gpu": gpus, + }, + }, + "env": [ + {"name": "NVIDIA_VISIBLE_DEVICES", "value": "all"}, + ], + } +} + +gpu_container_no_env(name, image, gpus) = c { + c := { + "name": name, + "image": image, + "resources": { + "limits": { + "nvidia.com/gpu": gpus, + }, + }, + } +} + +gpu_container_no_env_zero(name, image) = c { + c := { + "name": name, + "image": image, + "resources": { + "limits": { + "nvidia.com/gpu": "0", + }, + }, + } +} + +no_gpu_container(name, image) = c { + c := { + "name": name, + "image": image, + "resources": { + "limits": { + "cpu": "100m", + "memory": "128Mi", + }, + }, + } +} diff --git a/src/general/requiredgpuruntimeclass/constraint.tmpl b/src/general/requiredgpuruntimeclass/constraint.tmpl new file mode 100644 index 000000000..7a1e3765f --- /dev/null +++ b/src/general/requiredgpuruntimeclass/constraint.tmpl @@ -0,0 +1,49 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8srequiredgpuruntimeclass + annotations: + metadata.gatekeeper.sh/title: "Required GPU Runtime Class" + metadata.gatekeeper.sh/version: 1.0.0 + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to specify + a runtimeClassName from an allowed list. This ensures GPU workloads use + the proper container runtime (e.g., nvidia) rather than relying on default + runtime hooks. +spec: + crd: + spec: + names: + kind: K8sRequiredGpuRuntimeClass + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to specify an allowed runtimeClassName. + properties: + allowedRuntimeClassNames: + description: >- + List of allowed runtime class names for GPU workloads (e.g., ["nvidia"]). + type: array + items: + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: +{{ file.Read "src/general/requiredgpuruntimeclass/src.cel" | strings.Indent 10 | strings.TrimSuffix "\n" }} + - engine: Rego + source: + rego: | +{{ file.Read "src/general/requiredgpuruntimeclass/src.rego" | strings.Indent 12 | strings.TrimSuffix "\n" }} + libs: + - | +{{ file.Read "src/general/requiredgpuruntimeclass/lib_exempt_container.rego" | strings.Indent 14 | strings.TrimSuffix "\n" }} diff --git a/src/general/requiredgpuruntimeclass/lib_exempt_container.rego b/src/general/requiredgpuruntimeclass/lib_exempt_container.rego new file mode 100644 index 000000000..43e84d75e --- /dev/null +++ b/src/general/requiredgpuruntimeclass/lib_exempt_container.rego @@ -0,0 +1,19 @@ +package lib.exempt_container + +is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) +} + +_matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img +} + +_matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) +} \ No newline at end of file diff --git a/src/general/requiredgpuruntimeclass/src.cel b/src/general/requiredgpuruntimeclass/src.cel new file mode 100644 index 000000000..b7ff84a95 --- /dev/null +++ b/src/general/requiredgpuruntimeclass/src.cel @@ -0,0 +1,39 @@ +variables: +- name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' +- name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' +- name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' +- name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) +- name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) +- name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) +- name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) +- name: allowedRuntimeClassNames + expression: 'has(variables.params.allowedRuntimeClassNames) ? variables.params.allowedRuntimeClassNames : []' +- name: hasAllowedRc + expression: | + !has(variables.anyObject.spec.runtimeClassName) ? false : + variables.anyObject.spec.runtimeClassName in variables.allowedRuntimeClassNames +validations: +- expression: '!variables.podRequestsGpu || size(variables.allowedRuntimeClassNames) == 0 || variables.hasAllowedRc' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not specify an allowed runtimeClassName (allowed: " + variables.allowedRuntimeClassNames.join(", ") + ")"' diff --git a/src/general/requiredgpuruntimeclass/src.rego b/src/general/requiredgpuruntimeclass/src.rego new file mode 100644 index 000000000..5b347d50b --- /dev/null +++ b/src/general/requiredgpuruntimeclass/src.rego @@ -0,0 +1,35 @@ +package k8srequiredgpuruntimeclass + +import data.lib.exempt_container.is_exempt + +violation[{"msg": msg}] { + pod_requests_gpu + allowed := object.get(input, ["parameters", "allowedRuntimeClassNames"], []) + count(allowed) > 0 + not has_allowed_runtime_class(allowed) + msg := sprintf("Pod <%v> requests GPU resources but does not specify an allowed runtimeClassName (allowed: %v)", [input.review.object.metadata.name, allowed]) +} + +pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +has_allowed_runtime_class(allowed) { + rc := input.review.object.spec.runtimeClassName + rc == allowed[_] +} + +input_containers[c] { + c := input.review.object.spec.containers[_] +} + +input_containers[c] { + c := input.review.object.spec.initContainers[_] +} + +input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] +} diff --git a/src/general/requiredgpuruntimeclass/src_test.rego b/src/general/requiredgpuruntimeclass/src_test.rego new file mode 100644 index 000000000..642f24ffa --- /dev/null +++ b/src/general/requiredgpuruntimeclass/src_test.rego @@ -0,0 +1,41 @@ +package k8srequiredgpuruntimeclass + +test_gpu_with_runtime_class_allowed { + inp := {"review": review_with_rc([gpu_container("train", "nvidia/cuda:12.0", "1")], "nvidia"), "parameters": {"allowedRuntimeClassNames": ["nvidia"]}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_without_runtime_class_denied { + inp := {"review": review([gpu_container("train", "nvidia/cuda:12.0", "1")]), "parameters": {"allowedRuntimeClassNames": ["nvidia"]}} + results := violation with input as inp + count(results) == 1 +} + +test_gpu_wrong_runtime_class_denied { + inp := {"review": review_with_rc([gpu_container("train", "nvidia/cuda:12.0", "1")], "runc"), "parameters": {"allowedRuntimeClassNames": ["nvidia"]}} + results := violation with input as inp + count(results) == 1 +} + +test_no_gpu_allowed { + inp := {"review": review([no_gpu_container("web", "nginx:latest")]), "parameters": {"allowedRuntimeClassNames": ["nvidia"]}} + results := violation with input as inp + count(results) == 0 +} + +review(containers) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers}}} +} + +review_with_rc(containers, rc) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers, "runtimeClassName": rc}}} +} + +gpu_container(name, image, gpus) = c { + c := {"name": name, "image": image, "resources": {"limits": {"nvidia.com/gpu": gpus}}} +} + +no_gpu_container(name, image) = c { + c := {"name": name, "image": image, "resources": {"limits": {"cpu": "100m"}}} +} diff --git a/src/general/requiredgputoleration/constraint.tmpl b/src/general/requiredgputoleration/constraint.tmpl new file mode 100644 index 000000000..877b7cfb7 --- /dev/null +++ b/src/general/requiredgputoleration/constraint.tmpl @@ -0,0 +1,47 @@ +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8srequiredgputoleration + annotations: + metadata.gatekeeper.sh/title: "Required GPU Toleration" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to include + a toleration for the specified GPU node taint. This ensures GPU workloads + are properly configured to schedule on GPU nodes. +spec: + crd: + spec: + names: + kind: K8sRequiredGpuToleration + validation: + openAPIV3Schema: + type: object + description: >- + Requires pods requesting GPUs to include a specified toleration. + properties: + tolerationKey: + description: >- + The taint key that GPU pods must tolerate (e.g., "nvidia.com/gpu"). + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: +{{ file.Read "src/general/requiredgputoleration/src.cel" | strings.Indent 10 | strings.TrimSuffix "\n" }} + - engine: Rego + source: + rego: | +{{ file.Read "src/general/requiredgputoleration/src.rego" | strings.Indent 12 | strings.TrimSuffix "\n" }} + libs: + - | +{{ file.Read "src/general/requiredgputoleration/lib_exempt_container.rego" | strings.Indent 14 | strings.TrimSuffix "\n" }} diff --git a/src/general/requiredgputoleration/lib_exempt_container.rego b/src/general/requiredgputoleration/lib_exempt_container.rego new file mode 100644 index 000000000..43e84d75e --- /dev/null +++ b/src/general/requiredgputoleration/lib_exempt_container.rego @@ -0,0 +1,19 @@ +package lib.exempt_container + +is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) +} + +_matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img +} + +_matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) +} \ No newline at end of file diff --git a/src/general/requiredgputoleration/src.cel b/src/general/requiredgputoleration/src.cel new file mode 100644 index 000000000..71ea43b99 --- /dev/null +++ b/src/general/requiredgputoleration/src.cel @@ -0,0 +1,39 @@ +variables: +- name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' +- name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' +- name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' +- name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) +- name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) +- name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) +- name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) +- name: tolerationKey + expression: 'has(variables.params.tolerationKey) ? variables.params.tolerationKey : ""' +- name: hasToleration + expression: | + !has(variables.anyObject.spec.tolerations) ? false : + variables.anyObject.spec.tolerations.exists(t, t.key == variables.tolerationKey) +validations: +- expression: '!variables.podRequestsGpu || variables.tolerationKey == "" || variables.hasToleration' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not tolerate taint key <" + variables.tolerationKey + ">"' diff --git a/src/general/requiredgputoleration/src.rego b/src/general/requiredgputoleration/src.rego new file mode 100644 index 000000000..c2c0546c6 --- /dev/null +++ b/src/general/requiredgputoleration/src.rego @@ -0,0 +1,35 @@ +package k8srequiredgputoleration + +import data.lib.exempt_container.is_exempt + +violation[{"msg": msg}] { + pod_requests_gpu + toleration_key := object.get(input, ["parameters", "tolerationKey"], "") + toleration_key != "" + not has_toleration(toleration_key) + msg := sprintf("Pod <%v> requests GPU resources but does not tolerate taint key <%v>", [input.review.object.metadata.name, toleration_key]) +} + +pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 +} + +has_toleration(key) { + toleration := input.review.object.spec.tolerations[_] + toleration.key == key +} + +input_containers[c] { + c := input.review.object.spec.containers[_] +} + +input_containers[c] { + c := input.review.object.spec.initContainers[_] +} + +input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] +} diff --git a/src/general/requiredgputoleration/src_test.rego b/src/general/requiredgputoleration/src_test.rego new file mode 100644 index 000000000..d5a5f442a --- /dev/null +++ b/src/general/requiredgputoleration/src_test.rego @@ -0,0 +1,41 @@ +package k8srequiredgputoleration + +test_gpu_with_toleration_allowed { + inp := {"review": review_with_tolerations([gpu_container("train", "nvidia/cuda:12.0", "1")], [{"key": "nvidia.com/gpu", "operator": "Exists", "effect": "NoSchedule"}]), "parameters": {"tolerationKey": "nvidia.com/gpu"}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_without_toleration_denied { + inp := {"review": review([gpu_container("train", "nvidia/cuda:12.0", "1")]), "parameters": {"tolerationKey": "nvidia.com/gpu"}} + results := violation with input as inp + count(results) == 1 +} + +test_no_gpu_allowed { + inp := {"review": review([no_gpu_container("web", "nginx:latest")]), "parameters": {"tolerationKey": "nvidia.com/gpu"}} + results := violation with input as inp + count(results) == 0 +} + +test_gpu_wrong_toleration_denied { + inp := {"review": review_with_tolerations([gpu_container("train", "nvidia/cuda:12.0", "1")], [{"key": "other-key", "operator": "Exists", "effect": "NoSchedule"}]), "parameters": {"tolerationKey": "nvidia.com/gpu"}} + results := violation with input as inp + count(results) == 1 +} + +review(containers) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers}}} +} + +review_with_tolerations(containers, tolerations) = output { + output = {"object": {"metadata": {"name": "test-pod"}, "spec": {"containers": containers, "tolerations": tolerations}}} +} + +gpu_container(name, image, gpus) = c { + c := {"name": name, "image": image, "resources": {"limits": {"nvidia.com/gpu": gpus}}} +} + +no_gpu_container(name, image) = c { + c := {"name": name, "image": image, "resources": {"limits": {"cpu": "100m"}}} +} diff --git a/src/pod-security-policy/privileged-containers/src_test.cel b/src/pod-security-policy/privileged-containers/src_test.cel new file mode 100644 index 000000000..6041957c3 --- /dev/null +++ b/src/pod-security-policy/privileged-containers/src_test.cel @@ -0,0 +1,259 @@ +tests: +# ---- Variable-level tests ---- + +- name: "containers extracts spec.containers" + variable: containers + object: + spec: + containers: + - name: nginx + image: nginx + expect: + size: 1 + +- name: "containers returns empty when field missing" + variable: containers + object: + spec: {} + expect: + value: [] + +- name: "initContainers extracts init containers" + variable: initContainers + object: + spec: + initContainers: + - name: init + image: busybox + expect: + size: 1 + +- name: "initContainers returns empty when missing" + variable: initContainers + object: + spec: {} + expect: + value: [] + +- name: "exemptImagePrefixes extracts wildcard prefixes" + variable: exemptImagePrefixes + params: + exemptImages: + - "registry.k8s.io/*" + - "exact-image:latest" + object: + spec: + containers: [] + expect: + value: + - "registry.k8s.io/" + +- name: "exemptImageExplicit extracts non-wildcard images" + variable: exemptImageExplicit + params: + exemptImages: + - "registry.k8s.io/*" + - "exact-image:latest" + object: + spec: + containers: [] + expect: + value: + - "exact-image:latest" + +- name: "exemptImages empty when no params" + variable: exemptImages + object: + spec: + containers: + - name: nginx + image: nginx + expect: + value: [] + +- name: "badContainers finds privileged container" + variable: badContainers + object: + spec: + containers: + - name: nginx + image: nginx + securityContext: + privileged: true + expect: + size: 1 + contains: "Privileged container is not allowed: nginx" + +- name: "badContainers empty for non-privileged" + variable: badContainers + object: + spec: + containers: + - name: nginx + image: nginx + securityContext: + privileged: false + expect: + value: [] + +- name: "badContainers skips exempt image" + variable: badContainers + params: + exemptImages: + - "nginx" + object: + spec: + containers: + - name: nginx + image: nginx + securityContext: + privileged: true + expect: + value: [] + +- name: "badContainers skips wildcard exempt" + variable: badContainers + params: + exemptImages: + - "registry.k8s.io/*" + object: + spec: + containers: + - name: special + image: registry.k8s.io/special:latest + securityContext: + privileged: true + expect: + value: [] + +- name: "badContainers skips no securityContext" + variable: badContainers + object: + spec: + containers: + - name: nginx + image: nginx + expect: + value: [] + +- name: "isUpdate false on CREATE" + variable: isUpdate + object: + spec: + containers: [] + expect: + value: false + +- name: "isUpdate true on UPDATE" + variable: isUpdate + request: + operation: UPDATE + object: + spec: + containers: [] + expect: + value: true + +# ---- Policy-level tests (whole validations) ---- + +- name: "denies privileged container" + object: + metadata: + name: test-pod + spec: + containers: + - name: nginx + image: nginx + securityContext: + privileged: true + expect: + allowed: false + messageContains: "Privileged container is not allowed: nginx" + +- name: "allows non-privileged container" + object: + metadata: + name: test-pod + spec: + containers: + - name: nginx + image: nginx + securityContext: + privileged: false + expect: + allowed: true + +- name: "allows when no securityContext" + object: + metadata: + name: test-pod + spec: + containers: + - name: nginx + image: nginx + expect: + allowed: true + +- name: "allows exempt image even if privileged" + params: + exemptImages: + - "registry.k8s.io/special:latest" + object: + metadata: + name: test-pod + spec: + containers: + - name: special + image: registry.k8s.io/special:latest + securityContext: + privileged: true + expect: + allowed: true + +- name: "allows wildcard exempt image" + params: + exemptImages: + - "registry.k8s.io/*" + object: + metadata: + name: test-pod + spec: + containers: + - name: special + image: registry.k8s.io/special:latest + securityContext: + privileged: true + expect: + allowed: true + +- name: "denies privileged init container" + object: + metadata: + name: test-pod + spec: + containers: + - name: app + image: app + initContainers: + - name: init + image: init + securityContext: + privileged: true + expect: + allowed: false + messageContains: "Privileged container is not allowed: init" + +- name: "allows on UPDATE regardless" + request: + operation: UPDATE + object: + metadata: + name: test-pod + spec: + containers: + - name: nginx + image: nginx + securityContext: + privileged: true + expect: + allowed: true diff --git a/test/bats/helpers.bash b/test/bats/helpers.bash index b76bb8e16..cf8494836 100644 --- a/test/bats/helpers.bash +++ b/test/bats/helpers.bash @@ -109,3 +109,47 @@ constraint_enforced() { echo "ready: ${ready_count}, expected: ${pod_count}" [[ "${ready_count}" -eq "${pod_count}" ]] } + +validating_admission_policy_available() { + kubectl api-resources --api-group=admissionregistration.k8s.io -o name | grep -qx "validatingadmissionpolicies" +} + +template_uses_k8s_native_validation() { + local policy="$1" + grep -q "engine: K8sNativeValidation" "${policy}/template.yaml" +} + +should_wait_for_vap_enforcement() { + local policy="$1" + [[ "${POLICY_ENGINE:-rego}" == "cel" ]] && \ + template_uses_k8s_native_validation "${policy}" && \ + validating_admission_policy_available +} + +constraint_vap_enforced() { + local kind="$1" + local name="$2" + local pod_list="$(kubectl -n gatekeeper-system get pod -l gatekeeper.sh/operation=webhook -o json)" + if [[ $? -ne 0 ]]; then + echo "error gathering pods" + return 1 + fi + + local pod_count=$(echo "${pod_list}" | jq '.items | length') + if [[ ${pod_count} -lt 1 ]]; then + echo "Gatekeeper pod count is < 1" + return 2 + fi + + local cstr="$(kubectl get ${kind} ${name} -ojson)" + if [[ $? -ne 0 ]]; then + echo "Error gathering constraint ${kind} ${name}" + return 3 + fi + + echo "checking VAP enforcement for constraint ${cstr}" + + local ready_count=$(echo "${cstr}" | jq '.metadata.generation as $generation | [.status.byPod[]? | select(.operations[]? == "webhook" and .observedGeneration == $generation) | .enforcementPointsStatus[]? | select(.enforcementPoint == "vap.k8s.io" and .state == "generated" and .observedGeneration == $generation)] | length') + echo "VAP ready: ${ready_count}, expected: ${pod_count}" + [[ "${ready_count}" -eq "${pod_count}" ]] +} diff --git a/test/bats/test.bats b/test/bats/test.bats index 3e9cd5df0..54bbf7111 100755 --- a/test/bats/test.bats +++ b/test/bats/test.bats @@ -95,6 +95,9 @@ setup() { wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "kubectl apply -f ${sample}/constraint.yaml" local name=$(yq e .metadata.name "$sample"/constraint.yaml) wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "constraint_enforced $kind $name" + if should_wait_for_vap_enforcement "$policy"; then + wait_for_process ${WAIT_TIME} ${SLEEP_TIME} "constraint_vap_enforced $kind $name" + fi for inventory in "$sample"/example_inventory*.yaml; do if [[ -e "$inventory" ]]; then diff --git a/website/docs/validation/gpuactivedeadline.md b/website/docs/validation/gpuactivedeadline.md new file mode 100644 index 000000000..734e11495 --- /dev/null +++ b/website/docs/validation/gpuactivedeadline.md @@ -0,0 +1,415 @@ +--- +id: gpuactivedeadline +title: GPU Active Deadline Required +--- + +# GPU Active Deadline Required + +**Bundles:** `gatekeeper-ai-training-policies` + +## Description +Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set activeDeadlineSeconds. This prevents runaway training jobs from holding GPU resources indefinitely. + +## Template +```yaml +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuactivedeadline + annotations: + metadata.gatekeeper.sh/title: "GPU Active Deadline Required" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-ai-training-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set + activeDeadlineSeconds. This prevents runaway training jobs from holding + GPU resources indefinitely. +spec: + crd: + spec: + names: + kind: K8sGpuActiveDeadline + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to set activeDeadlineSeconds. + properties: + maxActiveDeadlineSeconds: + description: >- + The maximum value allowed for activeDeadlineSeconds. Set to 0 to + only require the field is present without enforcing a maximum. + type: integer + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: hasDeadline + expression: 'has(variables.anyObject.spec.activeDeadlineSeconds)' + - name: maxDeadline + expression: 'has(variables.params.maxActiveDeadlineSeconds) ? variables.params.maxActiveDeadlineSeconds : 0' + validations: + - expression: '!variables.podRequestsGpu || variables.hasDeadline' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not set activeDeadlineSeconds"' + - expression: '!variables.podRequestsGpu || !variables.hasDeadline || variables.maxDeadline == 0 || variables.anyObject.spec.activeDeadlineSeconds <= variables.maxDeadline' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> sets activeDeadlineSeconds to " + string(variables.anyObject.spec.activeDeadlineSeconds) + ", which exceeds the maximum allowed " + string(variables.maxDeadline)' + - engine: Rego + source: + rego: | + package k8sgpuactivedeadline + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + not has_active_deadline + msg := sprintf("Pod <%v> requests GPU resources but does not set activeDeadlineSeconds", [input.review.object.metadata.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + has_active_deadline + max_deadline := object.get(input, ["parameters", "maxActiveDeadlineSeconds"], 0) + max_deadline > 0 + deadline := input.review.object.spec.activeDeadlineSeconds + deadline > max_deadline + msg := sprintf("Pod <%v> sets activeDeadlineSeconds to %v, which exceeds the maximum allowed %v", [input.review.object.metadata.name, deadline, max_deadline]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_active_deadline { + input.review.object.spec.activeDeadlineSeconds + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } + +``` + +### Usage +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/template.yaml +``` +## Examples +
+gpu-job-with-deadline + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxActiveDeadlineSeconds: 86400 + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-with-deadline +spec: + activeDeadlineSeconds: 3600 + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/example_allowed.yaml +``` + +
+
+example-disallowed-exceeds-max + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-exceeds-deadline +spec: + activeDeadlineSeconds: 172800 + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/gpu-job-with-deadline/example_disallowed_exceeds_max.yaml +``` + +
+ + +
+gpu-job-without-deadline + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/gpu-job-without-deadline/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-without-deadline +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/gpu-job-without-deadline/example_disallowed.yaml +``` + +
+ + +
+non-gpu-job + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/non-gpu-job/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-job +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/non-gpu-job/example_allowed.yaml +``` + +
+ + +
+gpu-job-exempt + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuActiveDeadline +metadata: + name: require-gpu-deadline +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxActiveDeadlineSeconds: 86400 + exemptImages: + - "nvidia/dcgm-exporter:*" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/gpu-job-exempt/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-job-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuactivedeadline/samples/gpu-job-exempt/example_allowed.yaml +``` + +
+ + +
\ No newline at end of file diff --git a/website/docs/validation/gpunodetargeting.md b/website/docs/validation/gpunodetargeting.md new file mode 100644 index 000000000..ea31d6fd4 --- /dev/null +++ b/website/docs/validation/gpunodetargeting.md @@ -0,0 +1,710 @@ +--- +id: gpunodetargeting +title: GPU Node Targeting +--- + +# GPU Node Targeting + +**Bundles:** `gatekeeper-gpu-safety-policies` `gatekeeper-ai-training-policies` `gatekeeper-ai-inference-policies` + +## Description +Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to target GPU-labeled nodes using required node affinity or nodeSelector. This helps ensure GPU workloads only land on nodes that advertise GPU capacity. + +## Template +```yaml +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpunodetargeting + annotations: + metadata.gatekeeper.sh/title: "GPU Node Targeting" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to target + GPU-labeled nodes using required node affinity or nodeSelector. This helps + ensure GPU workloads only land on nodes that advertise GPU capacity. +spec: + crd: + spec: + names: + kind: K8sGpuNodeTargeting + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to target nodes with a configured GPU label key and optional values. + properties: + nodeLabelKey: + description: >- + The node label key that GPU workloads must target (for example, `nvidia.com/gpu.present` + or `nvidia.com/gpu.product`). + type: string + nodeLabelValues: + description: >- + Optional allowed values for the GPU node label. If omitted, the policy only requires the + label key to be present. + type: array + items: + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: allContainers + expression: 'variables.containers + variables.initContainers + variables.ephemeralContainers' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.allContainers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + variables.allContainers.exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + ( + (has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) || + (has(container.resources.requests) && + "nvidia.com/gpu" in container.resources.requests && + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) + ) + ) + - name: nodeLabelKey + expression: 'has(variables.params.nodeLabelKey) ? variables.params.nodeLabelKey : ""' + - name: nodeLabelValues + expression: 'has(variables.params.nodeLabelValues) ? variables.params.nodeLabelValues : []' + - name: hasMatchingNodeSelector + expression: | + !has(variables.anyObject.spec.nodeSelector) || !(variables.nodeLabelKey in variables.anyObject.spec.nodeSelector) ? false : + (size(variables.nodeLabelValues) == 0 ? + string(variables.anyObject.spec.nodeSelector[variables.nodeLabelKey]) != "" : + variables.anyObject.spec.nodeSelector[variables.nodeLabelKey] in variables.nodeLabelValues) + - name: hasMatchingNodeAffinity + expression: | + !has(variables.anyObject.spec.affinity) || + !has(variables.anyObject.spec.affinity.nodeAffinity) || + !has(variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution) || + !has(variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms) ? false : + variables.anyObject.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms.exists(term, + has(term.matchExpressions) && + term.matchExpressions.exists(expr, + expr.key == variables.nodeLabelKey && + ( + size(variables.nodeLabelValues) == 0 ? + expr.operator == "Exists" : + expr.operator == "In" && + has(expr.values) && + size(expr.values) > 0 && + expr.values.all(exprValue, exprValue in variables.nodeLabelValues) + ) + ) + ) + validations: + - expression: '!variables.podRequestsGpu || variables.nodeLabelKey == "" || variables.hasMatchingNodeSelector || variables.hasMatchingNodeAffinity' + messageExpression: | + size(variables.nodeLabelValues) == 0 ? + "Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not target nodes with label key <" + variables.nodeLabelKey + "> using node affinity or nodeSelector" : + "Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not target nodes with label <" + variables.nodeLabelKey + "> matching one of <" + variables.nodeLabelValues.join(", ") + "> using node affinity or nodeSelector" + - engine: Rego + source: + rego: | + package k8sgpunodetargeting + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + label_key := object.get(input.parameters, "nodeLabelKey", "") + label_key != "" + not has_matching_node_selector(label_key) + not has_matching_node_affinity(label_key) + label_values := object.get(input.parameters, "nodeLabelValues", []) + msg := violation_message(label_key, label_values) + } + + violation_message(label_key, label_values) = msg { + count(label_values) == 0 + msg := sprintf("Pod <%v> requests GPU resources but does not target nodes with label key <%v> using node affinity or nodeSelector", [input.review.object.metadata.name, label_key]) + } + + violation_message(label_key, label_values) = msg { + count(label_values) > 0 + msg := sprintf("Pod <%v> requests GPU resources but does not target nodes with label <%v> matching one of <%v> using node affinity or nodeSelector", [input.review.object.metadata.name, label_key, label_values]) + } + + pod_requests_gpu { + container := all_containers[_] + not is_exempt(container) + requests_gpu(container) + } + + all_containers[c] { + c := input.review.object.spec.containers[_] + } + + all_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + all_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + + requests_gpu(container) { + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu := limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + requests_gpu(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + gpu := requests["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_matching_node_selector(label_key) { + selector := input.review.object.spec.nodeSelector + value := selector[label_key] + value != "" + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) == 0 + } + + has_matching_node_selector(label_key) { + selector := input.review.object.spec.nodeSelector + value := selector[label_key] + label_values := object.get(input.parameters, "nodeLabelValues", []) + label_values[_] == value + } + + has_matching_node_affinity(label_key) { + term := input.review.object.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[_] + expr := term.matchExpressions[_] + expr.key == label_key + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) == 0 + expr.operator == "Exists" + } + + has_matching_node_affinity(label_key) { + term := input.review.object.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms[_] + expr := term.matchExpressions[_] + expr.key == label_key + label_values := object.get(input.parameters, "nodeLabelValues", []) + count(label_values) > 0 + expr.operator == "In" + values := object.get(expr, "values", []) + count(values) > 0 + not has_disallowed_affinity_value(values, label_values) + } + + has_disallowed_affinity_value(values, label_values) { + value := values[_] + not allowed_affinity_value(value, label_values) + } + + allowed_affinity_value(value, label_values) { + label_values[_] == value + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } +``` + +### Usage +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/template.yaml +``` +## Examples +
+gpu-pod-with-node-affinity + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_allowed.yaml +``` + +
+
+example-disallowed-wrong-value + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-wrong-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "false" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_disallowed_wrong_value.yaml +``` + +
+
+example-disallowed-mixed-values + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-mixed-node-affinity +spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: nvidia.com/gpu.present + operator: In + values: + - "true" + - "false" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-affinity/example_disallowed_mixed_values.yaml +``` + +
+ + +
+gpu-pod-with-node-selector + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-selector/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-node-selector +spec: + nodeSelector: + nvidia.com/gpu.present: "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-with-node-selector/example_allowed.yaml +``` + +
+ + +
+gpu-pod-without-targeting + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-without-targeting/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-targeting +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-without-targeting/example_disallowed.yaml +``` + +
+ + +
+non-gpu-pod + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/non-gpu-pod/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/non-gpu-pod/example_allowed.yaml +``` + +
+ + +
+gpu-pod-node-selector-key-only + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-node-selector-key-only +spec: + nodeSelector: + nvidia.com/gpu.present: "true" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/example_allowed.yaml +``` + +
+
+example-disallowed-empty-value + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-node-selector-empty-value +spec: + nodeSelector: + nvidia.com/gpu.present: "" + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-node-selector-key-only/example_disallowed_empty_value.yaml +``` + +
+ + +
+gpu-pod-exempt + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuNodeTargeting +metadata: + name: require-gpu-node-targeting +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + nodeLabelKey: "nvidia.com/gpu.present" + nodeLabelValues: + - "true" + exemptImages: + - "nvidia/dcgm-exporter:*" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-exempt/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpunodetargeting/samples/gpu-pod-exempt/example_allowed.yaml +``` + +
+ + +
\ No newline at end of file diff --git a/website/docs/validation/gpuresourcelimits.md b/website/docs/validation/gpuresourcelimits.md new file mode 100644 index 000000000..00f4b22b5 --- /dev/null +++ b/website/docs/validation/gpuresourcelimits.md @@ -0,0 +1,484 @@ +--- +id: gpuresourcelimits +title: GPU Resource Limits +--- + +# GPU Resource Limits + +**Bundles:** `gatekeeper-gpu-safety-policies` `gatekeeper-ai-training-policies` `gatekeeper-ai-inference-policies` + +## Description +Enforces a maximum number of NVIDIA GPUs (nvidia.com/gpu) that a single container may request. This prevents individual containers from hoarding GPU resources on shared clusters, particularly for AI/ML training workloads. + +## Template +```yaml +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuresourcelimits + annotations: + metadata.gatekeeper.sh/title: "GPU Resource Limits" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Enforces a maximum number of NVIDIA GPUs (nvidia.com/gpu) that a single + container may request. This prevents individual containers from hoarding + GPU resources on shared clusters, particularly for AI/ML training workloads. +spec: + crd: + spec: + names: + kind: K8sGpuResourceLimits + validation: + openAPIV3Schema: + type: object + description: >- + Enforces a maximum number of NVIDIA GPUs per container. + properties: + maxGpuPerContainer: + description: >- + The maximum number of GPUs a single container may request. + type: integer + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: maxGpu + expression: 'has(variables.params.maxGpuPerContainer) ? variables.params.maxGpuPerContainer : 0' + - name: badContainers + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + variables.maxGpu > 0 && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity(string(variables.maxGpu))) > 0 + ).map(container, "Container <" + container.name + "> requests " + string(container.resources.limits["nvidia.com/gpu"]) + " GPUs, which exceeds the maximum allowed " + string(variables.maxGpu)) + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpuresourcelimits + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input_containers[_] + not is_exempt(container) + gpu_count := to_number(container.resources.limits["nvidia.com/gpu"]) + gpu_count > 0 + max_gpu := object.get(input, ["parameters", "maxGpuPerContainer"], 0) + max_gpu > 0 + gpu_count > max_gpu + msg := sprintf("Container <%v> requests %v GPUs, which exceeds the maximum allowed %v", [container.name, gpu_count, max_gpu]) + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } + +``` + +### Usage +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/template.yaml +``` +## Examples +
+gpu-within-limit + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/gpu-within-limit/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-within-limit +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "2" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/gpu-within-limit/example_allowed.yaml +``` + +
+ + +
+gpu-exceeds-limit + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exceeds-limit +spec: + containers: + - name: training + image: myrepo/large-training:v1 + resources: + limits: + nvidia.com/gpu: "8" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/gpu-exceeds-limit/example_disallowed.yaml +``` + +
+ + +
+non-gpu-pod + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/non-gpu-pod/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/non-gpu-pod/example_allowed.yaml +``` + +
+ + +
+gpu-exempt-over-limit + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 + exemptImages: + - "nvidia/dcgm-exporter:*" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/gpu-exempt-over-limit/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-over-limit +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "8" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/gpu-exempt-over-limit/example_allowed.yaml +``` + +
+ + +
+gpu-max-disabled + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/gpu-max-disabled/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-max-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "8" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/gpu-max-disabled/example_allowed.yaml +``` + +
+ + +
+init-gpu-exceeds-limit + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuResourceLimits +metadata: + name: max-gpu-per-container +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + maxGpuPerContainer: 4 +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/init-gpu-exceeds-limit/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: init-gpu-exceeds-limit +spec: + initContainers: + - name: setup + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "8" + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuresourcelimits/samples/init-gpu-exceeds-limit/example_disallowed.yaml +``` + +
+ + +
\ No newline at end of file diff --git a/website/docs/validation/gpusharedmemory.md b/website/docs/validation/gpusharedmemory.md new file mode 100644 index 000000000..cbf710844 --- /dev/null +++ b/website/docs/validation/gpusharedmemory.md @@ -0,0 +1,492 @@ +--- +id: gpusharedmemory +title: GPU Shared Memory Required +--- + +# GPU Shared Memory Required + +**Bundles:** `gatekeeper-ai-training-policies` + +## Description +Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to mount a memory-backed emptyDir volume at /dev/shm. PyTorch DataLoader, NCCL multi-GPU communication, and most training frameworks require shared memory beyond the default 64MB. + +## Template +```yaml +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpusharedmemory + annotations: + metadata.gatekeeper.sh/title: "GPU Shared Memory Required" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-ai-training-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to mount a + memory-backed emptyDir volume at /dev/shm. PyTorch DataLoader, NCCL + multi-GPU communication, and most training frameworks require shared memory + beyond the default 64MB. +spec: + crd: + spec: + names: + kind: K8sGpuSharedMemory + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to mount a memory-backed volume at /dev/shm. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.containers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: volumes + expression: 'has(variables.anyObject.spec.volumes) ? variables.anyObject.spec.volumes : []' + - name: memoryVolNames + expression: | + variables.volumes.filter(v, + has(v.emptyDir) && has(v.emptyDir.medium) && v.emptyDir.medium == "Memory" + ).map(v, v.name) + - name: badContainers + expression: | + variables.containers.filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + (!has(container.volumeMounts) || + !container.volumeMounts.exists(vm, + vm.mountPath == "/dev/shm" && + vm.name in variables.memoryVolNames + ) + ) + ).map(container, "Container <" + container.name + "> requests GPU resources but does not mount a memory-backed volume at /dev/shm") + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpusharedmemory + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input.review.object.spec.containers[_] + not is_exempt(container) + has_gpu_request(container) + not has_shm_mount(container) + msg := sprintf("Container <%v> requests GPU resources but does not mount a memory-backed volume at /dev/shm", [container.name]) + } + + has_gpu_request(container) { + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_shm_mount(container) { + mount := container.volumeMounts[_] + mount.mountPath == "/dev/shm" + volume := input.review.object.spec.volumes[_] + volume.name == mount.name + volume.emptyDir.medium == "Memory" + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } + +``` + +### Usage +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/template.yaml +``` +## Examples +
+gpu-with-shm + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-with-shm/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-shm +spec: + volumes: + - name: dshm + emptyDir: + medium: Memory + sizeLimit: 8Gi + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /dev/shm + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-with-shm/example_allowed.yaml +``` + +
+ + +
+gpu-without-shm + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-without-shm/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-shm +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-without-shm/example_disallowed.yaml +``` + +
+ + +
+no-gpu + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/no-gpu/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/no-gpu/example_allowed.yaml +``` + +
+ + +
+gpu-shm-non-memory + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-shm-non-memory/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-shm-non-memory +spec: + volumes: + - name: dshm + emptyDir: {} + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /dev/shm +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-shm-non-memory/example_disallowed.yaml +``` + +
+ + +
+gpu-shm-wrong-path + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-shm-wrong-path/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-shm-wrong-path +spec: + volumes: + - name: dshm + emptyDir: + medium: Memory + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + volumeMounts: + - name: dshm + mountPath: /cache +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-shm-wrong-path/example_disallowed.yaml +``` + +
+ + +
+gpu-exempt-without-shm + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuSharedMemory +metadata: + name: require-gpu-shm +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-exempt-without-shm/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-shm +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpusharedmemory/samples/gpu-exempt-without-shm/example_allowed.yaml +``` + +
+ + +
\ No newline at end of file diff --git a/website/docs/validation/gpuworkloadresources.md b/website/docs/validation/gpuworkloadresources.md new file mode 100644 index 000000000..fd2c84852 --- /dev/null +++ b/website/docs/validation/gpuworkloadresources.md @@ -0,0 +1,799 @@ +--- +id: gpuworkloadresources +title: GPU Workload Resources +--- + +# GPU Workload Resources + +**Bundles:** `gatekeeper-gpu-safety-policies` `gatekeeper-ai-training-policies` `gatekeeper-ai-inference-policies` + +## Description +Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set memory requests equal to limits for all non-exempt containers and to set a CPU request. Containers that request GPUs must also set matching GPU requests and limits. This keeps GPU workloads schedulable and predictable while still allowing CPU burst above request. + +## Template +```yaml +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8sgpuworkloadresources + annotations: + metadata.gatekeeper.sh/title: "GPU Workload Resources" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to set + memory requests equal to limits for all non-exempt containers and to set + a CPU request. Containers that request GPUs must also set matching GPU + requests and limits. This keeps GPU workloads schedulable and predictable + while still allowing CPU burst above request. +spec: + crd: + spec: + names: + kind: K8sGpuWorkloadResources + validation: + openAPIV3Schema: + type: object + description: >- + Enforces memory request/limit alignment and cpu requests for GPU pods, + plus matching gpu requests and limits on GPU containers. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: enforcedContainers + expression: 'variables.containers + variables.initContainers' + - name: allContainers + expression: 'variables.enforcedContainers + variables.ephemeralContainers' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + variables.allContainers.filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: gpuContainers + expression: | + variables.allContainers.filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + ( + (has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) || + (has(container.resources.requests) && + "nvidia.com/gpu" in container.resources.requests && + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) > 0) + ) + ) + - name: podRequestsGpu + expression: 'size(variables.gpuContainers) > 0' + - name: gpuRequestLimitMessages + expression: | + variables.gpuContainers.filter(container, + !has(container.resources.requests) || + !("nvidia.com/gpu" in container.resources.requests) || + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity("0")) <= 0 || + !has(container.resources.limits) || + !("nvidia.com/gpu" in container.resources.limits) || + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) <= 0 || + quantity(string(container.resources.requests["nvidia.com/gpu"])).compareTo(quantity(string(container.resources.limits["nvidia.com/gpu"]))) != 0 + ).map(container, "Container <" + container.name + "> must set nvidia.com/gpu request equal to limit") + - name: memoryRequestLimitMessages + expression: | + !variables.podRequestsGpu ? [] : + variables.enforcedContainers.filter(container, + !(container.image in variables.exemptImages) && + ( + !has(container.resources) || + !has(container.resources.requests) || + !("memory" in container.resources.requests) || + !has(container.resources.limits) || + !("memory" in container.resources.limits) || + quantity(string(container.resources.requests["memory"])).compareTo(quantity(string(container.resources.limits["memory"]))) != 0 + ) + ).map(container, "Container <" + container.name + "> in a GPU pod must set memory request equal to limit") + - name: cpuRequestMessages + expression: | + !variables.podRequestsGpu ? [] : + variables.enforcedContainers.filter(container, + !(container.image in variables.exemptImages) && + ( + !has(container.resources) || + !has(container.resources.requests) || + !("cpu" in container.resources.requests) || + string(container.resources.requests["cpu"]) == "" + ) + ).map(container, "Container <" + container.name + "> in a GPU pod must set a cpu request") + validations: + - expression: 'size(variables.gpuRequestLimitMessages) == 0' + messageExpression: 'variables.gpuRequestLimitMessages.join(", ")' + - expression: 'size(variables.memoryRequestLimitMessages) == 0' + messageExpression: 'variables.memoryRequestLimitMessages.join(", ")' + - expression: 'size(variables.cpuRequestMessages) == 0' + messageExpression: 'variables.cpuRequestMessages.join(", ")' + - engine: Rego + source: + rego: | + package k8sgpuworkloadresources + + import data.lib.exempt_container.is_exempt + + missing(obj, field) = true { + not obj[field] + } + + missing(obj, field) = true { + obj[field] == "" + } + + # 10 ** 21 + mem_multiple("E") = 1000000000000000000000 { true } + + # 10 ** 18 + mem_multiple("P") = 1000000000000000000 { true } + + # 10 ** 15 + mem_multiple("T") = 1000000000000000 { true } + + # 10 ** 12 + mem_multiple("G") = 1000000000000 { true } + + # 10 ** 9 + mem_multiple("M") = 1000000000 { true } + + # 10 ** 6 + mem_multiple("k") = 1000000 { true } + + # 10 ** 3 + mem_multiple("") = 1000 { true } + + # Kubernetes accepts millibyte precision when it probably shouldn't. + # https://github.com/kubernetes/kubernetes/issues/28741 + # 10 ** 0 + mem_multiple("m") = 1 { true } + + # 1000 * 2 ** 10 + mem_multiple("Ki") = 1024000 { true } + + # 1000 * 2 ** 20 + mem_multiple("Mi") = 1048576000 { true } + + # 1000 * 2 ** 30 + mem_multiple("Gi") = 1073741824000 { true } + + # 1000 * 2 ** 40 + mem_multiple("Ti") = 1099511627776000 { true } + + # 1000 * 2 ** 50 + mem_multiple("Pi") = 1125899906842624000 { true } + + # 1000 * 2 ** 60 + mem_multiple("Ei") = 1152921504606846976000 { true } + + get_suffix(mem) = suffix { + not is_string(mem) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 0 + suffix := substring(mem, count(mem) - 1, -1) + mem_multiple(suffix) + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 1 + suffix := substring(mem, count(mem) - 2, -1) + mem_multiple(suffix) + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) > 1 + not mem_multiple(substring(mem, count(mem) - 1, -1)) + not mem_multiple(substring(mem, count(mem) - 2, -1)) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) == 1 + not mem_multiple(substring(mem, count(mem) - 1, -1)) + suffix := "" + } + + get_suffix(mem) = suffix { + is_string(mem) + count(mem) == 0 + suffix := "" + } + + canonify_mem(orig) = new { + is_number(orig) + new := orig * 1000 + } + + canonify_mem(orig) = new { + not is_number(orig) + suffix := get_suffix(orig) + raw := replace(orig, suffix, "") + regex.match("^[0-9]+(\\.[0-9]+)?$", raw) + new := to_number(raw) * mem_multiple(suffix) + } + + violation[{"msg": msg}] { + container := gpu_containers[_] + not has_matching_gpu_request_and_limit(container) + msg := sprintf("Container <%v> must set nvidia.com/gpu request equal to limit", [container.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + container := enforced_containers[_] + not is_exempt(container) + not has_matching_memory_request_and_limit(container) + msg := sprintf("Container <%v> in a GPU pod must set memory request equal to limit", [container.name]) + } + + violation[{"msg": msg}] { + pod_requests_gpu + container := enforced_containers[_] + not is_exempt(container) + not has_cpu_request(container) + msg := sprintf("Container <%v> in a GPU pod must set a cpu request", [container.name]) + } + + pod_requests_gpu { + gpu_containers[_] + } + + gpu_containers[c] { + c := all_containers[_] + not is_exempt(c) + requests_gpu(c) + } + + enforced_containers[c] { + c := input.review.object.spec.containers[_] + } + + enforced_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + all_containers[c] { + c := enforced_containers[_] + } + + all_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + + requests_gpu(container) { + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu := limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + requests_gpu(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + gpu := requests["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_matching_gpu_request_and_limit(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + limits := object.get(object.get(container, "resources", {}), "limits", {}) + gpu_request := requests["nvidia.com/gpu"] + gpu_limit := limits["nvidia.com/gpu"] + to_number(gpu_request) > 0 + to_number(gpu_limit) > 0 + to_number(gpu_request) == to_number(gpu_limit) + } + + has_matching_memory_request_and_limit(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + limits := object.get(object.get(container, "resources", {}), "limits", {}) + mem_request := requests["memory"] + mem_limit := limits["memory"] + canonify_mem(mem_request) == canonify_mem(mem_limit) + } + + has_cpu_request(container) { + requests := object.get(object.get(container, "resources", {}), "requests", {}) + not missing(requests, "cpu") + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } +``` + +### Usage +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/template.yaml +``` +## Examples +
+gpu-pod-compliant + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-compliant/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-compliant +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-compliant/example_allowed.yaml +``` + +
+ + +
+gpu-pod-memory-mismatch + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-memory-mismatch/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-memory-mismatch +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "8Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-memory-mismatch/example_disallowed.yaml +``` + +
+ + +
+gpu-pod-cpu-request-missing + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-cpu-request-missing/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-cpu-request-missing +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "4" + - name: logger + image: busybox:1.36 + resources: + requests: + memory: "256Mi" + limits: + memory: "256Mi" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-cpu-request-missing/example_disallowed.yaml +``` + +
+ + +
+non-gpu-pod + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/non-gpu-pod/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: non-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/non-gpu-pod/example_allowed.yaml +``` + +
+ + +
+gpu-pod-gpu-request-missing + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-gpu-request-missing/constraint.yaml +``` + +
+ + + +
+gpu-pod-gpu-mismatch + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-gpu-mismatch/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-gpu-mismatch +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "16Gi" + cpu: "2" + limits: + nvidia.com/gpu: "2" + memory: "16Gi" + cpu: "4" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-gpu-mismatch/example_disallowed.yaml +``` + +
+ + +
+gpu-pod-memory-equivalent + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-memory-equivalent/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-memory-equivalent +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + requests: + nvidia.com/gpu: "1" + memory: "1024Mi" + cpu: "2" + limits: + nvidia.com/gpu: "1" + memory: "1Gi" + cpu: "4" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-memory-equivalent/example_allowed.yaml +``` + +
+ + +
+gpu-pod-exempt + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sGpuWorkloadResources +metadata: + name: require-gpu-workload-resources +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-exempt/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/gpuworkloadresources/samples/gpu-pod-exempt/example_allowed.yaml +``` + +
+ + +
\ No newline at end of file diff --git a/website/docs/validation/nounsupportedgpu.md b/website/docs/validation/nounsupportedgpu.md new file mode 100644 index 000000000..9ac320fd9 --- /dev/null +++ b/website/docs/validation/nounsupportedgpu.md @@ -0,0 +1,516 @@ +--- +id: nounsupportedgpu +title: No Unsupported GPU Requests +--- + +# No Unsupported GPU Requests + +## Description +Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set the NVIDIA_VISIBLE_DEVICES environment variable, indicating the container image is built to consume GPUs via the NVIDIA CUDA runtime. This prevents GPU resource waste from containers that request GPUs but cannot use them. + +## Template +```yaml +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8snounsupportedgpu + annotations: + metadata.gatekeeper.sh/title: "No Unsupported GPU Requests" + metadata.gatekeeper.sh/version: 1.0.0 + description: >- + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set + the NVIDIA_VISIBLE_DEVICES environment variable, indicating the container + image is built to consume GPUs via the NVIDIA CUDA runtime. This prevents + GPU resource waste from containers that request GPUs but cannot use them. +spec: + crd: + spec: + names: + kind: K8sNoUnsupportedGpu + validation: + openAPIV3Schema: + type: object + description: >- + Containers which request NVIDIA GPU resources (nvidia.com/gpu) must set + the NVIDIA_VISIBLE_DEVICES environment variable. + properties: + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. For example: `my-image-*`. + + It is recommended that users use the fully-qualified Docker image name (e.g. start with a domain name) + in order to avoid unexpectedly exempting images from an untrusted repository. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: badContainers + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 && + (!has(container.env) || !container.env.exists(e, e.name == "NVIDIA_VISIBLE_DEVICES")) + ).map(container, "Container <" + container.name + "> requests nvidia.com/gpu but does not set the NVIDIA_VISIBLE_DEVICES environment variable") + validations: + - expression: 'size(variables.badContainers) == 0' + messageExpression: 'variables.badContainers.join(", ")' + - engine: Rego + source: + rego: | + package k8snounsupportedgpu + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + container := input_containers[_] + not is_exempt(container) + has_gpu_request(container) + not has_nvidia_env(container) + msg := sprintf("Container <%v> requests nvidia.com/gpu but does not set the NVIDIA_VISIBLE_DEVICES environment variable", [container.name]) + } + + has_gpu_request(container) { + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_nvidia_env(container) { + env := container.env[_] + env.name == "NVIDIA_VISIBLE_DEVICES" + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } + +``` + +### Usage +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/template.yaml +``` +## Examples +
+gpu-with-env-var + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-with-env-var/constraint.yaml +``` + +
+ +
+example-allowed-gpu-with-env + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-allowed +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + env: + - name: NVIDIA_VISIBLE_DEVICES + value: all + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-with-env-var/example_allowed.yaml +``` + +
+ + +
+gpu-without-env-var + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:*" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-without-env-var/constraint.yaml +``` + +
+ +
+example-disallowed-gpu-no-env + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-disallowed +spec: + containers: + - name: training + image: myrepo/custom-ml-image:v1 + resources: + limits: + nvidia.com/gpu: "1" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-without-env-var/example_disallowed.yaml +``` + +
+
+example-allowed-exempt-image + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-without-env-var/example_allowed_exempt.yaml +``` + +
+ + +
+no-gpu-requested + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/no-gpu-requested/constraint.yaml +``` + +
+ +
+example-allowed-no-gpu + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/no-gpu-requested/example_allowed.yaml +``` + +
+ + +
+gpu-init-without-env-var + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-init-without-env-var/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-init-without-env-var +spec: + initContainers: + - name: setup + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-init-without-env-var/example_disallowed.yaml +``` + +
+ + +
+gpu-ephemeral-without-env-var + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-ephemeral-without-env-var/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-ephemeral-without-env-var +spec: + containers: + - name: app + image: nginx:1.25 + resources: + limits: + cpu: "500m" + ephemeralContainers: + - name: debug-gpu + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-ephemeral-without-env-var/example_disallowed.yaml +``` + +
+ + +
+gpu-exact-exempt + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sNoUnsupportedGpu +metadata: + name: require-gpu-env-var +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + exemptImages: + - "nvidia/dcgm-exporter:3.1.7" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-exact-exempt/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exact-exempt +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/nounsupportedgpu/samples/gpu-exact-exempt/example_allowed.yaml +``` + +
+ + +
\ No newline at end of file diff --git a/website/docs/validation/requiredgpuruntimeclass.md b/website/docs/validation/requiredgpuruntimeclass.md new file mode 100644 index 000000000..894b719a5 --- /dev/null +++ b/website/docs/validation/requiredgpuruntimeclass.md @@ -0,0 +1,498 @@ +--- +id: requiredgpuruntimeclass +title: Required GPU Runtime Class +--- + +# Required GPU Runtime Class + +## Description +Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to specify a runtimeClassName from an allowed list. This ensures GPU workloads use the proper container runtime (e.g., nvidia) rather than relying on default runtime hooks. + +## Template +```yaml +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8srequiredgpuruntimeclass + annotations: + metadata.gatekeeper.sh/title: "Required GPU Runtime Class" + metadata.gatekeeper.sh/version: 1.0.0 + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to specify + a runtimeClassName from an allowed list. This ensures GPU workloads use + the proper container runtime (e.g., nvidia) rather than relying on default + runtime hooks. +spec: + crd: + spec: + names: + kind: K8sRequiredGpuRuntimeClass + validation: + openAPIV3Schema: + type: object + description: >- + Requires GPU pods to specify an allowed runtimeClassName. + properties: + allowedRuntimeClassNames: + description: >- + List of allowed runtime class names for GPU workloads (e.g., ["nvidia"]). + type: array + items: + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: allowedRuntimeClassNames + expression: 'has(variables.params.allowedRuntimeClassNames) ? variables.params.allowedRuntimeClassNames : []' + - name: hasAllowedRc + expression: | + !has(variables.anyObject.spec.runtimeClassName) ? false : + variables.anyObject.spec.runtimeClassName in variables.allowedRuntimeClassNames + validations: + - expression: '!variables.podRequestsGpu || size(variables.allowedRuntimeClassNames) == 0 || variables.hasAllowedRc' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not specify an allowed runtimeClassName (allowed: " + variables.allowedRuntimeClassNames.join(", ") + ")"' + - engine: Rego + source: + rego: | + package k8srequiredgpuruntimeclass + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + allowed := object.get(input, ["parameters", "allowedRuntimeClassNames"], []) + count(allowed) > 0 + not has_allowed_runtime_class(allowed) + msg := sprintf("Pod <%v> requests GPU resources but does not specify an allowed runtimeClassName (allowed: %v)", [input.review.object.metadata.name, allowed]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_allowed_runtime_class(allowed) { + rc := input.review.object.spec.runtimeClassName + rc == allowed[_] + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } + +``` + +### Usage +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/template.yaml +``` +## Examples +
+gpu-with-runtimeclass + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-with-runtimeclass/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-runtime +spec: + runtimeClassName: nvidia + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-with-runtimeclass/example_allowed.yaml +``` + +
+ + +
+gpu-without-runtimeclass + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-without-runtimeclass/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-runtime +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-without-runtimeclass/example_disallowed.yaml +``` + +
+ + +
+no-gpu + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/no-gpu/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/no-gpu/example_allowed.yaml +``` + +
+ + +
+gpu-wrong-runtimeclass + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-wrong-runtimeclass/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-wrong-runtimeclass +spec: + runtimeClassName: runc + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-wrong-runtimeclass/example_disallowed.yaml +``` + +
+ + +
+gpu-runtimeclass-disabled + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-runtimeclass-disabled/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-runtimeclass-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-runtimeclass-disabled/example_allowed.yaml +``` + +
+ + +
+gpu-exempt-without-runtimeclass + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuRuntimeClass +metadata: + name: require-gpu-runtime +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + allowedRuntimeClassNames: + - "nvidia" + exemptImages: + - "nvidia/dcgm-exporter:*" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-exempt-without-runtimeclass/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-runtimeclass +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgpuruntimeclass/samples/gpu-exempt-without-runtimeclass/example_allowed.yaml +``` + +
+ + +
\ No newline at end of file diff --git a/website/docs/validation/requiredgputoleration.md b/website/docs/validation/requiredgputoleration.md new file mode 100644 index 000000000..0c93888d3 --- /dev/null +++ b/website/docs/validation/requiredgputoleration.md @@ -0,0 +1,558 @@ +--- +id: requiredgputoleration +title: Required GPU Toleration +--- + +# Required GPU Toleration + +**Bundles:** `gatekeeper-gpu-safety-policies` `gatekeeper-ai-training-policies` `gatekeeper-ai-inference-policies` + +## Description +Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to include a toleration for the specified GPU node taint. This ensures GPU workloads are properly configured to schedule on GPU nodes. + +## Template +```yaml +apiVersion: templates.gatekeeper.sh/v1 +kind: ConstraintTemplate +metadata: + name: k8srequiredgputoleration + annotations: + metadata.gatekeeper.sh/title: "Required GPU Toleration" + metadata.gatekeeper.sh/version: 1.0.0 + metadata.gatekeeper.sh/bundle: "gatekeeper-gpu-safety-policies, gatekeeper-ai-training-policies, gatekeeper-ai-inference-policies" + description: >- + Requires pods that request NVIDIA GPU resources (nvidia.com/gpu) to include + a toleration for the specified GPU node taint. This ensures GPU workloads + are properly configured to schedule on GPU nodes. +spec: + crd: + spec: + names: + kind: K8sRequiredGpuToleration + validation: + openAPIV3Schema: + type: object + description: >- + Requires pods requesting GPUs to include a specified toleration. + properties: + tolerationKey: + description: >- + The taint key that GPU pods must tolerate (e.g., "nvidia.com/gpu"). + type: string + exemptImages: + description: >- + Any container that uses an image that matches an entry in this list will be excluded + from enforcement. Prefix-matching can be signified with `*`. + type: array + items: + type: string + targets: + - target: admission.k8s.gatekeeper.sh + code: + - engine: K8sNativeValidation + source: + variables: + - name: containers + expression: 'has(variables.anyObject.spec.containers) ? variables.anyObject.spec.containers : []' + - name: initContainers + expression: 'has(variables.anyObject.spec.initContainers) ? variables.anyObject.spec.initContainers : []' + - name: ephemeralContainers + expression: 'has(variables.anyObject.spec.ephemeralContainers) ? variables.anyObject.spec.ephemeralContainers : []' + - name: exemptImagePrefixes + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, image.endsWith("*")).map(image, string(image).replace("*", "")) + - name: exemptImageExplicit + expression: | + !has(variables.params.exemptImages) ? [] : + variables.params.exemptImages.filter(image, !image.endsWith("*")) + - name: exemptImages + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).filter(container, + container.image in variables.exemptImageExplicit || + variables.exemptImagePrefixes.exists(exemption, string(container.image).startsWith(exemption)) + ).map(container, container.image) + - name: podRequestsGpu + expression: | + (variables.containers + variables.initContainers + variables.ephemeralContainers).exists(container, + !(container.image in variables.exemptImages) && + has(container.resources) && + has(container.resources.limits) && + "nvidia.com/gpu" in container.resources.limits && + quantity(string(container.resources.limits["nvidia.com/gpu"])).compareTo(quantity("0")) > 0 + ) + - name: tolerationKey + expression: 'has(variables.params.tolerationKey) ? variables.params.tolerationKey : ""' + - name: hasToleration + expression: | + !has(variables.anyObject.spec.tolerations) ? false : + variables.anyObject.spec.tolerations.exists(t, t.key == variables.tolerationKey) + validations: + - expression: '!variables.podRequestsGpu || variables.tolerationKey == "" || variables.hasToleration' + messageExpression: '"Pod <" + variables.anyObject.metadata.name + "> requests GPU resources but does not tolerate taint key <" + variables.tolerationKey + ">"' + - engine: Rego + source: + rego: | + package k8srequiredgputoleration + + import data.lib.exempt_container.is_exempt + + violation[{"msg": msg}] { + pod_requests_gpu + toleration_key := object.get(input, ["parameters", "tolerationKey"], "") + toleration_key != "" + not has_toleration(toleration_key) + msg := sprintf("Pod <%v> requests GPU resources but does not tolerate taint key <%v>", [input.review.object.metadata.name, toleration_key]) + } + + pod_requests_gpu { + container := input_containers[_] + not is_exempt(container) + gpu := container.resources.limits["nvidia.com/gpu"] + to_number(gpu) > 0 + } + + has_toleration(key) { + toleration := input.review.object.spec.tolerations[_] + toleration.key == key + } + + input_containers[c] { + c := input.review.object.spec.containers[_] + } + + input_containers[c] { + c := input.review.object.spec.initContainers[_] + } + + input_containers[c] { + c := input.review.object.spec.ephemeralContainers[_] + } + libs: + - | + package lib.exempt_container + + is_exempt(container) { + exempt_images := object.get(object.get(input, "parameters", {}), "exemptImages", []) + img := container.image + exemption := exempt_images[_] + _matches_exemption(img, exemption) + } + + _matches_exemption(img, exemption) { + not endswith(exemption, "*") + exemption == img + } + + _matches_exemption(img, exemption) { + endswith(exemption, "*") + prefix := trim_suffix(exemption, "*") + startswith(img, prefix) + } + +``` + +### Usage +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/template.yaml +``` +## Examples +
+gpu-with-toleration + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-with-toleration/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-with-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "nvidia.com/gpu" + operator: "Exists" + effect: "NoSchedule" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-with-toleration/example_allowed.yaml +``` + +
+ + +
+gpu-without-toleration + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-without-toleration/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-pod-without-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-without-toleration/example_disallowed.yaml +``` + +
+ + +
+no-gpu + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/no-gpu/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: no-gpu-pod +spec: + containers: + - name: web + image: nginx:1.25 + resources: + limits: + cpu: "500m" + memory: "128Mi" + +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/no-gpu/example_allowed.yaml +``` + +
+ + +
+gpu-wrong-toleration + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-wrong-toleration/constraint.yaml +``` + +
+ +
+example-disallowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-wrong-toleration +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "workload" + operator: "Exists" + effect: "NoSchedule" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-wrong-toleration/example_disallowed.yaml +``` + +
+ + +
+gpu-toleration-disabled + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-toleration-disabled/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-toleration-disabled +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-toleration-disabled/example_allowed.yaml +``` + +
+ + +
+gpu-exempt-without-toleration + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" + exemptImages: + - "nvidia/dcgm-exporter:*" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-exempt-without-toleration/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-exempt-without-toleration +spec: + containers: + - name: dcgm + image: nvidia/dcgm-exporter:3.1.7 + resources: + limits: + nvidia.com/gpu: "1" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-exempt-without-toleration/example_allowed.yaml +``` + +
+ + +
+gpu-toleration-key-only + +
+constraint + +```yaml +apiVersion: constraints.gatekeeper.sh/v1beta1 +kind: K8sRequiredGpuToleration +metadata: + name: require-gpu-toleration +spec: + match: + kinds: + - apiGroups: [""] + kinds: ["Pod"] + parameters: + tolerationKey: "nvidia.com/gpu" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-toleration-key-only/constraint.yaml +``` + +
+ +
+example-allowed + +```yaml +apiVersion: v1 +kind: Pod +metadata: + name: gpu-toleration-key-only +spec: + containers: + - name: training + image: nvidia/cuda:12.0-runtime + resources: + limits: + nvidia.com/gpu: "1" + tolerations: + - key: "nvidia.com/gpu" + operator: "Equal" + value: "present" + effect: "NoExecute" +``` + +Usage + +```shell +kubectl apply -f https://raw.githubusercontent.com/open-policy-agent/gatekeeper-library/master/library/general/requiredgputoleration/samples/gpu-toleration-key-only/example_allowed.yaml +``` + +
+ + +
\ No newline at end of file diff --git a/website/sidebars.js b/website/sidebars.js index c0ae2e0b8..1b91f28e7 100644 --- a/website/sidebars.js +++ b/website/sidebars.js @@ -50,6 +50,64 @@ module.exports = { 'validation/verifydeprecatedapi', ], }, + { + type: 'category', + label: 'AI Workload Policies', + collapsed: true, + items: [ + { + type: 'category', + label: 'Profiles', + collapsed: true, + items: [ + { + type: 'category', + label: 'GPU Safety', + collapsed: true, + items: [ + 'validation/gpunodetargeting', + 'validation/gpuresourcelimits', + 'validation/gpuworkloadresources', + 'validation/requiredgputoleration', + ], + }, + { + type: 'category', + label: 'Training', + collapsed: true, + items: [ + 'validation/gpuactivedeadline', + 'validation/gpunodetargeting', + 'validation/gpuresourcelimits', + 'validation/gpusharedmemory', + 'validation/gpuworkloadresources', + 'validation/requiredgputoleration', + ], + }, + { + type: 'category', + label: 'Inference', + collapsed: true, + items: [ + 'validation/gpunodetargeting', + 'validation/gpuresourcelimits', + 'validation/gpuworkloadresources', + 'validation/requiredgputoleration', + ], + }, + ], + }, + { + type: 'category', + label: 'Other', + collapsed: true, + items: [ + 'validation/nounsupportedgpu', + 'validation/requiredgpuruntimeclass', + ], + }, + ], + }, { type: 'category', label: 'Pod Security Standards',