diff --git a/docs/developer_guide/architecture.md b/docs/developer_guide/architecture.md index dd6a326..eb09b6e 100644 --- a/docs/developer_guide/architecture.md +++ b/docs/developer_guide/architecture.md @@ -52,8 +52,13 @@ nixidy/ ├── pkgs/ # Nix packages and generators │ └── generators/ # CRD and K8s schema generators │ ├── crd2jsonschema.py # CRD to JSON schema converter -│ ├── default.nix # Generator entry point -│ ├── generator.nix # Nix options generator +│ ├── default.nix # Generator entry point (accessors) +│ ├── walk.nix # Shared schema traversal (backend-parameterized) +│ ├── backend-text.nix # Text backend — emits Nix source +│ ├── backend-value.nix # Value backend — live module values +│ ├── generator.nix # File assembler (text backend → .nix file) +│ ├── module.nix # Module assembler (value backend → module value) +│ ├── runtime.nix # Runtime helpers as values (for module.nix) │ └── versions.nix # Kubernetes versions config ├── tests/ # Module unit tests │ ├── helm/ # Helm-specific tests @@ -355,38 +360,58 @@ Nixidy generates typed Nix options from: Output: `modules/generated/k8s/v1.XX.nix` +### Shared schema walk and backends + +The schema-to-options logic lives in one place, `walk.nix`, parameterized over a *backend* so the same traversal can produce either Nix **source text** or live module **values**: + +- `walk.nix` — the single traversal of the (JSON) schema. All type/coercion branch logic (int-or-string, `additionalProperties`→`attrsOf`, coerce-by-name lists, patch-merge-key, `skipCoerceToList`, `specialMapKeys`, nested submodules, etc.) lives here, expressed against an abstract backend `b`. +- `backend-text.nix` — emits Nix **source**. Used by the file generators; it also inlines the runtime helpers (see below) as source so committed standalone files stay self-contained. +- `backend-value.nix` — produces live module **values** using `runtime.nix`. +- `generator.nix` — thin assembler over the text backend (renders + `nixfmt` + writes a `.nix` file). +- `module.nix` — thin assembler over the value backend (returns a module function). +- `runtime.nix` — the per-generated-module runtime helpers (`coercedTo`, `mergeValuesByKey`, `submoduleForDefinition`, …) as **values**, imported by the value backend. The text backend inlines source-form equivalents of these same helpers. Only the *walk* is single-sourced; the helpers must exist in both forms because live values can't be serialized back to source and committed files can't reference nixidy internals. + ### CRD Generation -Two entry points: +CRD accessors form a matrix over output shape × source, all built on the shared walk: + +| | source files (`src`) | Helm chart | +| --- | --- | --- | +| → generated file | `fromCRD` | `fromChartCRD` | +| → module value | `fromCRDModule` | `fromChartCRDModule` | +| → raw objects | `crdObjects` | `crdObjectsFromChart` | -#### `fromCRD` ```nix fromCRD { name = "cilium"; src = pkgs.fetchFromGitHub { ... }; - crds = [ "path/to/crd.yaml" ]; + crdFiles = [ "path/to/crd.yaml" ]; # list of YAML files under `src` + kindFilter = [ ]; # Optional: only these CRD kinds (default: all) namePrefix = ""; # Optional prefix for attribute names attrNameOverrides = { }; # Manual name overrides skipCoerceToList = { }; # Skip list coercion for specific fields } -``` -#### `fromChartCRD` -```nix fromChartCRD { name = "cert-manager"; chartAttrs = { repo = "..."; chart = "..."; version = "..."; }; - crds = [ "Certificate" ]; # Filter by kind + kindFilter = [ "Certificate" ]; # Optional: only these CRD kinds (default: all) + kubeVersion = "v1.31.0"; # Optional: helm template --kube-version } ``` +The `*Module` variants take the same arguments and return a module value (no file); `crdObjects`/`crdObjectsFromChart` return the raw `CustomResourceDefinition` manifests as values. + +!!! note "Renamed arguments" + The overloaded `crds` argument was split: source-based accessors now take `crdFiles` (the list of YAML files) and chart-based accessors take `kindFilter` (the kind filter). `crds` remains as a deprecated alias on every accessor (resolved by the shared `renamedArg` helper, which warns and points at the new name). + ### CRD Processing (`crd2jsonschema.py`) Python script that: 1. Reads CRD YAML files 2. Extracts OpenAPI v3 schemas 3. Flattens `$ref` references -4. Outputs JSON schema for `generator.nix` +4. Outputs JSON schema for the shared walk (`walk.nix`) ## Testing Framework diff --git a/docs/llms.txt b/docs/llms.txt index 913b46d..13dbd1f 100644 --- a/docs/llms.txt +++ b/docs/llms.txt @@ -12,7 +12,7 @@ Nixidy brings the power of Nix to Kubernetes cluster management, enabling declar ## User Guide - [Using Helm Charts](https://nixidy.dev/user_guide/helm_charts/): Integrate existing Helm charts with patch support -- [Typed Resource Options](https://nixidy.dev/user_guide/typed_resources/): Generate typed Nix options from CRDs using fromCRD and fromChartCRD +- [Typed Resource Options](https://nixidy.dev/user_guide/typed_resources/): Generate typed Nix options from CRDs — fromCRD/fromChartCRD (generated file), fromCRDModule/fromChartCRDModule (module value, no file), and crdObjects/crdObjectsFromChart (raw CRD manifests) - [Templates](https://nixidy.dev/user_guide/templates/): Create reusable application patterns with configurable options - [Git Strategies](https://nixidy.dev/user_guide/git_strategies/): Monorepo, environment branches, or separate repositories - [GitHub Actions](https://nixidy.dev/user_guide/github_actions/): CI/CD integration with nixidy/build and nixidy/switch actions diff --git a/docs/user_guide/typed_resources.md b/docs/user_guide/typed_resources.md index 2658de8..6b7f9d0 100644 --- a/docs/user_guide/typed_resources.md +++ b/docs/user_guide/typed_resources.md @@ -34,6 +34,22 @@ Each file's basename becomes the output filename. Basenames must be unique withi Thankfully a code generator for generating resource options from CRDs is provided by nixidy (this is based heavily on kubenix's code generator). +nixidy offers a small matrix of CRD accessors. Pick one by **what you want back** (a generated `.nix` file, a live module value, or the raw CRD manifests) and **where the CRDs come from** (local source files, or a Helm chart): + +| | source files (`src`) | Helm chart | +| ------------------ | -------------------- | --------------------- | +| **→ generated file** | [`fromCRD`](#from-source-files-fromcrd) | [`fromChartCRD`](#from-helm-chart-crds-fromchartcrd) | +| **→ module value** | [`fromCRDModule`](#as-a-module-value-fromcrdmodule-fromchartcrdmodule) | [`fromChartCRDModule`](#as-a-module-value-fromcrdmodule-fromchartcrdmodule) | +| **→ raw objects** | [`crdObjects`](#raw-crd-objects-crdobjects-crdobjectsfromchart) | [`crdObjectsFromChart`](#raw-crd-objects-crdobjects-crdobjectsfromchart) | + +- **→ generated file** renders the resource options to Nix source, formats it, and writes a `.nix` file you commit and `import`. Good when you want the generated types checked into your repository and reviewable in diffs. +- **→ module value** returns the resource options as a module value directly — no generated file, no `import`. Drop it straight into `nixidy.applicationImports`. Good for flake/programmatic consumers that don't need a committed file. +- **→ raw objects** returns the `CustomResourceDefinition` manifests themselves as Nix values (not resource options), e.g. so you can deploy the CRDs alongside the resources that use them. + +All variants share the same `name`, `namePrefix`, `attrNameOverrides` and `skipCoerceToList` arguments; the source-based ones take `crdFiles` and the chart-based ones take `chartAttrs`/`chart`/`values`/`kubeVersion`. Both families take an optional `kindFilter` to narrow which CRD kinds are processed. + +### From source files (`fromCRD`) + As an example, to generate resource options for Cilium's `CiliumNetworkPolicy` and `CiliumClusterwideNetworkPolicy` the following can be defined in `flake.nix`. ```nix title="flake.nix" @@ -64,7 +80,7 @@ As an example, to generate resource options for Cilium's `CiliumNetworkPolicy` a rev = "v1.15.6"; hash = "sha256-oC6pjtiS8HvqzzRQsE+2bm6JP7Y3cbupXxCKSvP6/kU="; }; - crds = [ + crdFiles = [ "pkg/k8s/apis/cilium.io/client/crds/v2/ciliumnetworkpolicies.yaml" "pkg/k8s/apis/cilium.io/client/crds/v2/ciliumclusterwidenetworkpolicies.yaml" ]; @@ -113,7 +129,7 @@ Then running `nix build .#generators.cilium` will produce a nix file that can be rev = "v1.15.6"; hash = "sha256-oC6pjtiS8HvqzzRQsE+2bm6JP7Y3cbupXxCKSvP6/kU="; }; - crds = [ + crdFiles = [ "pkg/k8s/apis/cilium.io/client/crds/v2/ciliumnetworkpolicies.yaml" "pkg/k8s/apis/cilium.io/client/crds/v2/ciliumclusterwidenetworkpolicies.yaml" ]; @@ -123,7 +139,7 @@ Then running `nix build .#generators.cilium` will produce a nix file that can be Then running `nix-build generate.nix -A cilium` will produce a nix file that can be copied into place in your repository and imported using `nixidy.applicationImports` in a nixidy module. -### Generating resource options from Helm Chart CRDs +### From Helm Chart CRDs (`fromChartCRD`) In some cases, CRDs are only available through Helm charts or it's beneficial to keep them in sync with the chart version you're deploying. The `fromChartCRD` function provides a solution by templating the Helm chart and extracting CRDs from the output, generating nixidy resource options from them. @@ -166,7 +182,7 @@ As an example, to generate resource options for cert-manager's `Certificate` CRD }; # Or from nixhelm # chart = nixhelm.chartsDerivations.${system}.jetstack.cert-manager; - crds = [ "Certificate" ]; # Optional: filter by specific CRD kinds + kindFilter = [ "Certificate" ]; # Optional: filter by specific CRD kinds extraOpts = [ "--set", "crds.enabled=true"]; # Optional: pass extra options to helm template generation }; }; @@ -205,7 +221,7 @@ Then running `nix build .#generators.certManager` will produce a nix file that c version = "v1.19.1"; chartHash = "sha256-fs14wuKK+blC0l+pRfa//oBV2X+Dr3nNX+Z94nrQVrA="; }; - crds = [ "Certificate" ]; # Optional: filter by specific CRD kinds + kindFilter = [ "Certificate" ]; # Optional: filter by specific CRD kinds extraOpts = [ "--set", "crds.enabled=true"]; # Optional: pass extra options to helm template generation }; } @@ -218,12 +234,72 @@ The `fromChartCRD` function accepts the same optional arguments as `fromCRD` (`n - `chartAttrs`: Chart repository, name, version and chartHash configuration - `chart`: Alternative to `chartAttrs`, can use a pre-downloaded chart - `values`: Values to pass to the Helm chart templating -- `crds`: List of CRD kinds to extract (empty list extracts all CRDs) +- `kindFilter`: List of CRD kinds to extract (empty list extracts all CRDs) +- `kubeVersion`: Kubernetes version to template the chart against (`helm template --kube-version`). Defaults to the version in nixidy's nixpkgs; set it to match the cluster the CRDs are destined for. - `extraOpts`: List of extra arguments to pass through to helm template +!!! note "Renamed argument" + `kindFilter` was previously called `crds`. The old name still works as a deprecated alias (it emits a warning pointing at `kindFilter`); prefer `kindFilter` in new code. Note that on `fromCRD` the `crds` argument instead became `crdFiles` — the two used to share a name despite meaning different things (a kind filter vs. a list of YAML files). This approach ensures your CRD definitions stay synchronized with the Helm chart version you're actually deploying in your applications. +### As a module value (`fromCRDModule` / `fromChartCRDModule`) + +`fromCRD` and `fromChartCRD` build a `.nix` file that you commit and `import`. For flake or other programmatic consumers that don't need a committed file, `fromCRDModule` (and its chart counterpart `fromChartCRDModule`) return the resource options as a **module value** instead — skipping the render-to-source, format, write-file and `import` round-trip. The result is a module function, which is exactly what `nixidy.applicationImports` accepts, so it drops straight in. + +```nix title="flake.nix" +nixidyEnvs = nixidy.lib.mkEnvs { + inherit pkgs; + + envs.dev.modules = [ + ./env/dev.nix + { + nixidy.applicationImports = [ + (nixidy.packages.${system}.generators.fromCRDModule { + name = "cilium"; + src = pkgs.fetchFromGitHub { + owner = "cilium"; + repo = "cilium"; + rev = "v1.15.6"; + hash = "sha256-oC6pjtiS8HvqzzRQsE+2bm6JP7Y3cbupXxCKSvP6/kU="; + }; + crdFiles = [ + "pkg/k8s/apis/cilium.io/client/crds/v2/ciliumnetworkpolicies.yaml" + "pkg/k8s/apis/cilium.io/client/crds/v2/ciliumclusterwidenetworkpolicies.yaml" + ]; + }) + ]; + } + ]; +}; +``` + +`fromChartCRDModule` takes the same chart arguments as `fromChartCRD` (`chartAttrs`/`chart`/`values`/`kindFilter`/`kubeVersion`/`extraOpts`). Because no file is generated there is nothing to `nix build` or commit — the types are produced during evaluation. + +### Raw CRD objects (`crdObjects` / `crdObjectsFromChart`) + +Sometimes you don't want resource *options* at all — you want the `CustomResourceDefinition` manifests themselves, for example to deploy the CRDs into the cluster alongside the resources that use them. `crdObjects` (source files) and `crdObjectsFromChart` (Helm chart) return the CRD manifests as a list of plain Nix attribute sets. They are deployment-agnostic — what you do with them is up to you. + +One way to deploy them is to feed them into an application's `yamls`, which parses YAML/JSON strings into the application's output (`builtins.toJSON` output is valid YAML): + +```nix +# inside an env module +applications.crds = { + namespace = "kube-system"; + yamls = map builtins.toJSON ( + nixidy.packages.${system}.generators.crdObjects { + src = ciliumSrc; # the pkgs.fetchFromGitHub from the fromCRD example + crdFiles = [ + "pkg/k8s/apis/cilium.io/client/crds/v2/ciliumnetworkpolicies.yaml" + "pkg/k8s/apis/cilium.io/client/crds/v2/ciliumclusterwidenetworkpolicies.yaml" + ]; + } + ); +}; +``` + +`crdObjectsFromChart` takes the same chart arguments as `fromChartCRD`. Both accept an optional `kindFilter` to keep only specific CRD kinds (an empty or unset list keeps all of them). + ### Resolving Naming Conflicts Sometimes, multiple Custom Resource Definitions from different sources might define the same resource `kind`. This can lead to conflicts in the generated attribute names. For instance, if two different operators both define a CRD with the kind `Database`, they would both try to generate options for `resources.databases`. @@ -265,7 +341,7 @@ This argument is a mapping from the CRD's name (`.`) to the desir repo = "provider-keycloak"; ... }; - crds = [ + crdFiles = [ # This CRD conflicts with Kubernetes builtin Binding "package/crds/authenticationflow.keycloak.crossplane.io_bindings.yaml" # These CRDs have identical plural references diff --git a/flake.nix b/flake.nix index 94c6ed5..cd09754 100644 --- a/flake.nix +++ b/flake.nix @@ -168,6 +168,24 @@ '').outPath; }; + # Assert the CRD value accessors (fromCRDModule, crdObjects, and their + # chart variants) agree with the file generator and each other, and + # honor kubeVersion. The check derivation builds iff all pass. + crdAccessorTest = { + type = "app"; + program = + let + check = import ./pkgs/generators/equiv-test.nix { + inherit pkgs; + mkEnv = self.lib.mkEnv; + generators = self.packages.${system}.generators; + }; + in + (pkgs.writeShellScript "crd-accessor-test" '' + echo "CRD accessor checks passed (${check})" + '').outPath; + }; + # Serve docs docsServe = { type = "app"; diff --git a/pkgs/generators/backend-text.nix b/pkgs/generators/backend-text.nix new file mode 100644 index 0000000..044dff1 --- /dev/null +++ b/pkgs/generators/backend-text.nix @@ -0,0 +1,80 @@ +# Text backend for the shared schema walk (./walk.nix). +# +# Every combinator emits a fragment of Nix *source*. ./generator.nix assembles +# these into a standalone, committable `.nix` module file (the names referenced +# here — `submoduleOf`, `mkOption`, `types`, ... — are resolved by the runtime +# `let` block that generator.nix inlines into that file). +{ lib }: +with lib; +let + # Escape ${...} sequences so they appear as literal text in generated + # Nix "..." strings instead of being parsed as string interpolations. + escapeNixStr = str: builtins.replaceStrings [ "\${" ] [ "\\$\{" ] str; + + toNixString = + value: + if isAttrs value || isList value then + escapeNixStr (builtins.toJSON value) + else if isString value then + ''"${escapeNixStr value}"'' + else if value == null then + "null" + else + builtins.toString value; + + removeEmptyLines = + str: + concatStringsSep "\n" (filter (l: builtins.match "[[:space:]]*" l != [ ]) (splitString "\n" str)); +in +{ + types = { + unspecified = "types.unspecified"; + str = "types.str"; + int = "types.int"; + float = "types.float"; + bool = "types.bool"; + attrs = "types.attrs"; + nullOr = val: "(types.nullOr ${val})"; + attrsOf = val: "(types.attrsOf ${val})"; + listOf = val: "(types.listOf ${val})"; + either = val1: val2: "(types.either ${val1} ${val2})"; + loaOf = type: "(types.loaOf ${type})"; + # `walk` pre-maps the alternatives, so this receives rendered type strings. + oneOf = ts: "(types.oneOf [${concatStringsSep " " ts}])"; + }; + + mkOption = + { + description ? null, + type ? null, + default ? null, + apply ? null, + }: + removeEmptyLines '' + mkOption { + ${optionalString ( + description != null + ) "description = ${escapeNixStr (builtins.toJSON description)};"} + ${optionalString (type != null) "type = ${type};"} + ${optionalString (default != null) "default = ${toNixString default};"} + ${optionalString (apply != null) "apply = ${apply};"} + }''; + + mkOverrideNull = "mkOverride 1002 null"; + + submoduleOf = ref: ''(submoduleOf "${ref}")''; + + globalSubmoduleOf = ref: ''(globalSubmoduleOf "${ref}")''; + + submoduleForDefinition = + ref: name: kind: group: version: + ''(submoduleForDefinition "${ref}" "${name}" "${kind}" "${group}" "${version}")''; + + coerceAttrsOfSubmodulesToListByKey = + ref: attrMergeKey: listMergeKeys: + ''(coerceAttrsOfSubmodulesToListByKey "${ref}" "${attrMergeKey}" [${ + concatStringsSep " " (map (key: "\"${toString key}\"") listMergeKeys) + }])''; + + attrsToList = "attrsToList"; +} diff --git a/pkgs/generators/backend-value.nix b/pkgs/generators/backend-value.nix new file mode 100644 index 0000000..1915fea --- /dev/null +++ b/pkgs/generators/backend-value.nix @@ -0,0 +1,22 @@ +# Value backend for the shared schema walk (./walk.nix). +# +# Every combinator returns a live value (an option, a type, a submodule). It +# simply adapts the runtime helpers (./runtime.nix, `rt`) and `lib` into the +# backend interface the walk expects. ./module.nix assembles the walk output +# into a module value directly — no source, no file, no import IFD. +{ lib, rt }: +{ + # rt.types is `lib.types` with the custom `str`/`coercedTo` overrides the + # generated modules rely on; it already provides int/listOf/oneOf/loaOf/etc. + inherit (rt) + types + submoduleOf + globalSubmoduleOf + coerceAttrsOfSubmodulesToListByKey + submoduleForDefinition + attrsToList + ; + + mkOption = lib.mkOption; + mkOverrideNull = lib.mkOverride 1002 null; +} diff --git a/pkgs/generators/default.nix b/pkgs/generators/default.nix index 3ab6fed..781263c 100644 --- a/pkgs/generators/default.nix +++ b/pkgs/generators/default.nix @@ -176,25 +176,57 @@ let ######### ## CRD ## ######### - fromCRD = + # Resolve a renamed argument: prefer the new name, fall back to the + # deprecated `crds` alias (emitting a warning that points at the new name), + # else the supplied default. `default` is only forced when neither is given, + # so passing a `throw` makes the new argument effectively required. + renamedArg = + { + fn, + old ? "crds", + new, + newVal, + oldVal, + default, + }: + if newVal != null then + lib.warnIf ( + oldVal != null + ) "${fn}: both `${old}` and `${new}` given; ignoring deprecated `${old}`" newVal + else if oldVal != null then + lib.warn "${fn}: argument `${old}` is deprecated, use `${new}` instead" oldVal + else + default; + + # Run the CRD YAML through crd2jsonschema.py and read the resulting JSON + # schema back into Nix. + # + # The nix code generator is slightly modified from kubenix's generator. As + # it kind of depends on the jsonschema to be flattened with `$ref`s we first + # pre-process the CRD with a crude python script to flatten it before running + # the generator. See: crd2jsonschema.py + # + # This Python parse is the one unavoidable IFD; both the file generator + # (`fromCRD`) and the native module generator (`fromCRDModule`) share it. + crdSchema = { name, src, - crds, + crdFiles, namePrefix ? "", attrNameOverrides ? { }, - skipCoerceToList ? { }, # Optional list of CRD `kind` names to generate. When empty (the - # default) every CustomResourceDefinition found in `crds` is generated. - # Useful when `crds` points at a multi-document stream (e.g. raw - # `helm template` output) containing more kinds than you want. + # default) every CustomResourceDefinition found in `crdFiles` is + # generated. Useful when `crdFiles` points at a multi-document stream + # (e.g. raw `helm template` output) containing more kinds than you want. kindFilter ? [ ], }: let options = pkgs.writeText "${name}-crd2jsonschema-options.json" ( builtins.toJSON { + # crd2jsonschema.py reads this under the JSON key `crds`. + crds = crdFiles; inherit - crds namePrefix attrNameOverrides kindFilter @@ -202,16 +234,10 @@ let } ); - # The nix code generator is slightly modified from kubenix's - # generator. As it kind of depends on the jsonschema to be - # flattened with `$ref`s we first pre-process the CRD with - # a crude python script to flatten it before running the - # generator. - # See: crd2jsonschema.py - schema = - let - pythonWithYaml = pkgs.python3.withPackages (ps: [ ps.pyyaml ]); - in + pythonWithYaml = pkgs.python3.withPackages (ps: [ ps.pyyaml ]); + in + builtins.fromJSON ( + builtins.readFile ( pkgs.stdenv.mkDerivation { inherit src; @@ -225,8 +251,23 @@ let installPhase = '' ${pythonWithYaml}/bin/python ${./crd2jsonschema.py} "${options}" > $out ''; - }; - in + } + ) + ); + + fromCRD = + { + name, + src, + # List of CRD YAML files (relative to `src`) to generate types from. + crdFiles ? null, + # Deprecated alias for `crdFiles`. + crds ? null, + namePrefix ? "", + attrNameOverrides ? { }, + skipCoerceToList ? { }, + kindFilter ? [ ], + }: import ./generator.nix { inherit pkgs @@ -235,26 +276,144 @@ let skipCoerceToList ; - schema = builtins.fromJSON (builtins.readFile schema); + schema = crdSchema { + inherit + name + src + namePrefix + attrNameOverrides + kindFilter + ; + crdFiles = renamedArg { + fn = "fromCRD"; + new = "crdFiles"; + newVal = crdFiles; + oldVal = crds; + default = throw "fromCRD: `crdFiles` is required"; + }; + }; + }; + + # Like `fromCRD`, but returns the resource definitions as a module *value* + # (a `{ lib, options, config, ... }: { ... }` function) instead of a + # derivation that builds a `.nix` file. This removes the + # generate-source-then-`import` round-trip — the result can be placed + # directly in `nixidy.applicationImports`, which already accepts + # `functionTo attrs`. + fromCRDModule = + { + name, + src, + # List of CRD YAML files (relative to `src`) to generate types from. + crdFiles ? null, + # Deprecated alias for `crdFiles`. + crds ? null, + namePrefix ? "", + attrNameOverrides ? { }, + skipCoerceToList ? { }, + specialMapKeys ? { }, + kindFilter ? [ ], + }: + import ./module.nix { + inherit + lib + name + skipCoerceToList + specialMapKeys + ; + + schema = crdSchema { + inherit + name + src + namePrefix + attrNameOverrides + kindFilter + ; + crdFiles = renamedArg { + fn = "fromCRDModule"; + new = "crdFiles"; + newVal = crdFiles; + oldVal = crds; + default = throw "fromCRDModule: `crdFiles` is required"; + }; + }; }; + # Extract the raw CustomResourceDefinition objects from a set of CRD YAML + # files. The objects counterpart to `fromCRD`: same `src`/`crdFiles` inputs, + # but returns the CRD manifests as values (e.g. to apply them to a cluster) + # instead of generating resource option modules. Deployment-agnostic — what + # you do with the objects is up to you. + # + # `kindFilter`, when non-empty, keeps only CRDs whose `spec.names.kind` is in + # the list (mirrors `fromCRD`'s and `fromChartCRD`'s `kindFilter`). + crdObjects = + { + src, + # List of CRD YAML files (relative to `src`) to read. + crdFiles ? null, + # Deprecated alias for `crdFiles`. + crds ? null, + kindFilter ? [ ], + }: + let + files = renamedArg { + fn = "crdObjects"; + new = "crdFiles"; + newVal = crdFiles; + oldVal = crds; + default = throw "crdObjects: `crdFiles` is required"; + }; + + objects = lib.concatMap (f: klib.fromYAML (builtins.readFile "${src}/${f}")) files; + + isWanted = + obj: + obj != null + && obj ? kind + && obj.kind == "CustomResourceDefinition" + && (kindFilter == [ ] || lib.any (x: obj.spec.names.kind == x) kindFilter); + in + lib.filter isWanted objects; + fromChartCRD = { name, chartAttrs ? { }, chart ? null, values ? { }, - crds ? [ ], + # Optional list of CRD `kind` names to keep. Empty/unset = every CRD. + kindFilter ? null, + # Deprecated alias for `kindFilter`. + crds ? null, namePrefix ? "", attrNameOverrides ? { }, skipCoerceToList ? { }, extraOpts ? [ ], + # Kubernetes version to template the chart against (`helm template + # --kube-version`). Defaults to the version in nixidy's nixpkgs; override + # to match the cluster the CRDs are destined for. + kubeVersion ? "v${pkgs.kubernetes.version}", }: let + kindFilter' = renamedArg { + fn = "fromChartCRD"; + new = "kindFilter"; + newVal = kindFilter; + oldVal = crds; + default = [ ]; + }; + _chart = if chart != null then chart else klib.downloadHelmChart chartAttrs; objects = klib.fromHelm { - inherit name values extraOpts; + inherit + name + values + extraOpts + kubeVersion + ; includeCRDs = true; chart = _chart; }; @@ -263,7 +422,7 @@ let obj: obj ? kind && obj.kind == "CustomResourceDefinition" - && (crds == [ ] || (lib.any (x: obj.spec.names.kind == x) crds)); + && (kindFilter' == [ ] || (lib.any (x: obj.spec.names.kind == x) kindFilter')); filtered = lib.filter isWanted objects; @@ -287,13 +446,142 @@ let skipCoerceToList ; - crds = [ + crdFiles = [ "crds.yaml" ]; }; + + # Template a chart's CRDs to a raw `crds.yaml` derivation (helm template + # --include-crds). Shared by the chart-based module/object accessors so the + # chart is templated once; calling both with identical args reuses this one + # derivation. Unlike `fromChartCRD`, the output is the raw helm YAML (no + # re-serialization), which the downstream accessors parse directly. + mkChartCRDsYaml = + { + name, + chart ? null, + chartAttrs ? { }, + values ? { }, + extraOpts ? [ ], + kubeVersion ? "v${pkgs.kubernetes.version}", + }: + let + _chart = if chart != null then chart else klib.downloadHelmChart chartAttrs; + + templated = klib.buildHelmChart { + inherit + name + values + extraOpts + kubeVersion + ; + chart = _chart; + includeCRDs = true; + }; + in + # `buildHelmChart` emits a single YAML *file*; copy (not symlink) it into a + # directory `$out/crds.yaml`. `crdObjects` reads this at eval time via + # `readFile "${src}/crds.yaml"`, and a `linkFarm` symlink would make that + # read follow into a separate derivation output not carried in the string's + # context — forbidden in pure eval. A real file keeps the read in-context. + pkgs.runCommand "chart-crds-${name}" { } '' + mkdir -p $out + cp ${templated} $out/crds.yaml + ''; + + # Chart counterpart to `fromCRDModule`: template a chart's CRDs and return a + # module value (resource type options). `kindFilter`, when non-empty, narrows + # the generated types to those CRD kinds (mirrors `fromChartCRD`). + fromChartCRDModule = + { + name, + chart ? null, + chartAttrs ? { }, + values ? { }, + # Optional list of CRD `kind` names to keep. Empty/unset = every CRD. + kindFilter ? null, + # Deprecated alias for `kindFilter`. + crds ? null, + extraOpts ? [ ], + kubeVersion ? "v${pkgs.kubernetes.version}", + namePrefix ? "", + attrNameOverrides ? { }, + skipCoerceToList ? { }, + }: + fromCRDModule { + inherit + name + namePrefix + attrNameOverrides + skipCoerceToList + ; + src = mkChartCRDsYaml { + inherit + name + chart + chartAttrs + values + extraOpts + kubeVersion + ; + }; + crdFiles = [ "crds.yaml" ]; + kindFilter = renamedArg { + fn = "fromChartCRDModule"; + new = "kindFilter"; + newVal = kindFilter; + oldVal = crds; + default = [ ]; + }; + }; + + # Chart counterpart to `crdObjects`: template a chart's CRDs and return the + # raw CustomResourceDefinition manifests as values. `kindFilter` empty = every + # CRD. + crdObjectsFromChart = + { + name, + chart ? null, + chartAttrs ? { }, + values ? { }, + # Optional list of CRD `kind` names to keep. Empty/unset = every CRD. + kindFilter ? null, + # Deprecated alias for `kindFilter`. + crds ? null, + extraOpts ? [ ], + kubeVersion ? "v${pkgs.kubernetes.version}", + }: + crdObjects { + src = mkChartCRDsYaml { + inherit + name + chart + chartAttrs + values + extraOpts + kubeVersion + ; + }; + crdFiles = [ "crds.yaml" ]; + kindFilter = renamedArg { + fn = "crdObjectsFromChart"; + new = "kindFilter"; + newVal = kindFilter; + oldVal = crds; + default = [ ]; + }; + }; in { - inherit fromCRD fromChartCRD k8s; + inherit + fromCRD + fromCRDModule + fromChartCRD + fromChartCRDModule + crdObjects + crdObjectsFromChart + k8s + ; argocd = fromCRD { name = "argocd"; @@ -303,7 +591,7 @@ in rev = "v3.0.0"; hash = "sha256-g401mpNEhCNe8H6lk2HToAEZlZa16Py8ozK2z5/UozA="; }; - crds = [ + crdFiles = [ "manifests/crds/application-crd.yaml" "manifests/crds/applicationset-crd.yaml" "manifests/crds/appproject-crd.yaml" diff --git a/pkgs/generators/equiv-test.nix b/pkgs/generators/equiv-test.nix new file mode 100644 index 0000000..85ecab3 --- /dev/null +++ b/pkgs/generators/equiv-test.nix @@ -0,0 +1,336 @@ +# Tests for the CRD value accessors. Every accessor that returns Nix values +# must agree with the file generator and with its src/chart counterpart: +# +# - fromCRDModule renders identically to fromCRD (file) +# - fromChartCRDModule renders identically to fromCRDModule for the same CRD +# - crdObjects returns the raw CRD manifests (+ kindFilter narrows) +# - crdObjectsFromChart returns the same objects as crdObjects for the same CRD +# - the chart accessors honor `kubeVersion` +# +# Returns a derivation that builds iff every check passes. +{ + pkgs, + mkEnv, + generators, +}: +let + lib = pkgs.lib; + + # ── A FooBar CRD, as both a `src` tree and a local helm chart ────────────── + # Shaped to exercise the at-risk type/coercion branches: int-or-string + # (targetPort), additionalProperties→attrsOf (labels), coerce-by-name (ports), + # patch-merge-key (env), skipCoerceToList (volumes), nested submodule + # (resources), and the global ObjectMeta metadata ref. + crdYaml = '' + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: foobars.stable.example.com + spec: + group: stable.example.com + names: + kind: FooBar + plural: foobars + singular: foobar + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + properties: + image: + type: string + replicas: + type: integer + targetPort: + x-kubernetes-int-or-string: true + labels: + type: object + additionalProperties: + type: string + ports: + type: array + items: + type: object + properties: + name: + type: string + port: + type: integer + volumes: + type: array + items: + type: object + properties: + name: + type: string + path: + type: string + env: + type: array + x-kubernetes-patch-merge-key: key + items: + type: object + properties: + key: + type: string + value: + type: string + resources: + type: object + properties: + cpu: + type: string + memory: + type: string + status: + type: object + properties: + availableReplicas: + type: integer + ''; + + src = pkgs.runCommand "foobar-crd-src" { } '' + mkdir -p $out + cp ${pkgs.writeText "foobar.yaml" crdYaml} $out/foobar.yaml + ''; + + # Local helm chart shipping the same CRD in crds/ (helm --include-crds copies + # crds/ verbatim, so the chart path must match the src path exactly). + chart = pkgs.runCommand "foobar-chart" { } '' + mkdir -p $out/crds + cp ${pkgs.writeText "Chart.yaml" '' + apiVersion: v2 + name: foobar + version: 0.0.0 + ''} $out/Chart.yaml + cp ${pkgs.writeText "foobar.yaml" crdYaml} $out/crds/foobar.yaml + ''; + + # A chart whose CRD is *templated* (in templates/, not crds/) and embeds the + # helm `.Capabilities.KubeVersion` — so a passed kubeVersion is observable. + verChart = pkgs.runCommand "ver-chart" { } '' + mkdir -p $out/templates + cp ${pkgs.writeText "Chart.yaml" '' + apiVersion: v2 + name: ver + version: 0.0.0 + ''} $out/Chart.yaml + cp ${pkgs.writeText "crd.yaml" '' + apiVersion: apiextensions.k8s.io/v1 + kind: CustomResourceDefinition + metadata: + name: vers.test.example.com + annotations: + test/kube-version: "{{ .Capabilities.KubeVersion.Version }}" + spec: + group: test.example.com + names: + kind: Ver + plural: vers + singular: ver + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + ''} $out/templates/crd.yaml + ''; + + skipCoerceToList = { + "stable.example.com.v1.FooBarSpec" = [ "volumes" ]; + }; + + # ── The accessors under test ─────────────────────────────────────────────── + fileMod = import ( + generators.fromCRD { + name = "foobar"; + inherit src skipCoerceToList; + crdFiles = [ "foobar.yaml" ]; + } + ); + nativeMod = generators.fromCRDModule { + name = "foobar"; + inherit src skipCoerceToList; + crdFiles = [ "foobar.yaml" ]; + }; + chartMod = generators.fromChartCRDModule { + name = "foobar"; + inherit chart skipCoerceToList; + kindFilter = [ "FooBar" ]; + }; + + srcObjs = generators.crdObjects { + inherit src; + crdFiles = [ "foobar.yaml" ]; + }; + chartObjs = generators.crdObjectsFromChart { + name = "foobar"; + inherit chart; + }; + verObjs = + kubeVersion: + generators.crdObjectsFromChart { + name = "ver"; + chart = verChart; + inherit kubeVersion; + }; + + # Render a FooBar resource through a CRD type module (via mkEnv). Base imports + # (k8s core + argocd) provide ObjectMeta; the base doesn't define FooBar, so + # any difference is isolated to this resource. + render = + crdMod: + (mkEnv { + inherit pkgs; + modules = [ + { + nixidy.target = { + repository = "x"; + branch = "main"; + }; + nixidy.applicationImports = [ crdMod ]; + applications.test = { + namespace = "default"; + resources."stable.example.com"."v1"."FooBar".myfoo.spec = { + image = "nginx"; + replicas = 2; + targetPort = 8080; + labels.app = "demo"; + ports.http.port = 8080; + env.FOO.value = "bar"; + volumes = [ + { + name = "data"; + path = "/data"; + } + ]; + resources = { + cpu = "100m"; + memory = "128Mi"; + }; + }; + }; + } + ]; + }).config.applications.test.resources."stable.example.com"."v1"."FooBar".myfoo; + + # `specialMapKeys` is exposed only on the *module* accessors — `fromCRD` does + # not take it — so it can't be diffed against the file backend. Exercise the + # value backend's branch directly by rendering `spec.ports` as a *list* of + # name-less entries: coerceAttrsOfSubmodulesToListByKey synthesizes the merge + # key from `specialMapKeys` for entries lacking `name`, so distinct key values + # keep entries separate while the empty default collapses them. + renderPorts = + specialMapKeys: portsList: + (mkEnv { + inherit pkgs; + modules = [ + { + nixidy.target = { + repository = "x"; + branch = "main"; + }; + nixidy.applicationImports = [ + (generators.fromCRDModule { + name = "foobar"; + inherit src skipCoerceToList specialMapKeys; + crdFiles = [ "foobar.yaml" ]; + }) + ]; + applications.test = { + namespace = "default"; + resources."stable.example.com"."v1"."FooBar".myfoo.spec.ports = portsList; + }; + } + ]; + }).config.applications.test.resources."stable.example.com"."v1"."FooBar".myfoo.spec.ports; + + checks = { + "fromCRDModule == fromCRD (file)" = render nativeMod == render fileMod; + + "fromChartCRDModule == fromCRDModule" = render chartMod == render nativeMod; + + "crdObjects returns the CRD" = + lib.length srcObjs == 1 && (lib.head srcObjs).spec.names.kind == "FooBar"; + + "crdObjects kindFilter hit" = + lib.length ( + generators.crdObjects { + inherit src; + crdFiles = [ "foobar.yaml" ]; + kindFilter = [ "FooBar" ]; + } + ) == 1; + + "crdObjects kindFilter miss" = + generators.crdObjects { + inherit src; + crdFiles = [ "foobar.yaml" ]; + kindFilter = [ "Nope" ]; + } == [ ]; + + # Deprecated `crds` alias still resolves (src-based → crdFiles). + "crdObjects deprecated `crds` alias == `crdFiles`" = + generators.crdObjects { + inherit src; + crds = [ "foobar.yaml" ]; + } == srcObjs; + + # Deprecated `crds` alias still resolves (chart-based → kindFilter). + "crdObjectsFromChart deprecated `crds` alias == `kindFilter`" = + generators.crdObjectsFromChart { + name = "foobar"; + inherit chart; + crds = [ "FooBar" ]; + } == generators.crdObjectsFromChart { + name = "foobar"; + inherit chart; + kindFilter = [ "FooBar" ]; + }; + + # Value backend honors specialMapKeys (module-only arg). Two name-less ports + # with distinct `port` values: a `port` merge key keeps them separate (2); + # the empty default collapses them to one (1). + "fromCRDModule honors specialMapKeys (value backend)" = + let + portsList = [ + { port = 1; } + { port = 2; } + ]; + withKey = renderPorts { "stable.example.com.v1.FooBarSpec".ports = [ "port" ]; } portsList; + without = renderPorts { } portsList; + in + lib.length withKey == 2 && lib.length without == 1; + + "crdObjectsFromChart == crdObjects" = chartObjs == srcObjs; + + "crdObjectsFromChart honors kubeVersion" = + let + ann = kv: (lib.head (verObjs kv)).metadata.annotations."test/kube-version"; + in + lib.hasInfix "1.31" (ann "v1.31.0") + && lib.hasInfix "1.40" (ann "v1.40.0") + && ann "v1.31.0" != ann "v1.40.0"; + }; + + failed = lib.attrNames (lib.filterAttrs (_: ok: !ok) checks); +in +pkgs.runCommand "crd-accessor-test" { } ( + if failed == [ ] then + '' + echo "PASS: ${toString (lib.length (lib.attrNames checks))} CRD accessor checks" + touch $out + '' + else + throw "CRD accessor checks failed: ${lib.concatStringsSep "; " failed}" +) diff --git a/pkgs/generators/generator.nix b/pkgs/generators/generator.nix index eeb997a..29a7669 100644 --- a/pkgs/generators/generator.nix +++ b/pkgs/generators/generator.nix @@ -1,5 +1,11 @@ # This generator is based heavily on kubenix's generator. # See: https://github.com/hall/kubenix/blob/main/pkgs/generators/k8s/default.nix +# +# Text backend assembler: drives the shared schema walk (./walk.nix) with the +# text backend (./backend-text.nix) to render Nix source, then writes a +# standalone, committable `.nix` module file. The runtime helpers the generated +# file relies on are inlined into its `let` block below; ./runtime.nix is the +# value-form twin of those helpers used by the native module generator. { name, pkgs, @@ -11,346 +17,21 @@ }: with lib; let - applyOverlay = - overlay: attr: - let - f = final: attr; - in - lib.fix (lib.extends overlay f); - - # Escape ${...} sequences so they appear as literal text in generated - # Nix "..." strings instead of being parsed as string interpolations. - escapeNixStr = str: builtins.replaceStrings [ "\${" ] [ "\\$\{" ] str; - - gen = rec { - mkMerge = values: "mkMerge [${concatMapStrings (value: " - ${value} - ") values}]"; - - toNixString = - value: - if isAttrs value || isList value then - escapeNixStr (builtins.toJSON value) - else if isString value then - ''"${escapeNixStr value}"'' - else if value == null then - "null" - else - builtins.toString value; - - removeEmptyLines = - str: - concatStringsSep "\n" (filter (l: builtins.match "[[:space:]]*" l != [ ]) (splitString "\n" str)); - - mkOption = - { - description ? null, - type ? null, - default ? null, - apply ? null, - }: - removeEmptyLines '' - mkOption { - ${optionalString ( - description != null - ) "description = ${escapeNixStr (builtins.toJSON description)};"} - ${optionalString (type != null) "type = ${type};"} - ${optionalString (default != null) "default = ${toNixString default};"} - ${optionalString (apply != null) "apply = ${apply};"} - }''; - - mkOverride = priority: value: "mkOverride ${toString priority} ${toNixString value}"; - - types = { - unspecified = "types.unspecified"; - str = "types.str"; - int = "types.int"; - float = "types.float"; - bool = "types.bool"; - attrs = "types.attrs"; - nullOr = val: "(types.nullOr ${val})"; - attrsOf = val: "(types.attrsOf ${val})"; - listOf = val: "(types.listOf ${val})"; - coercedTo = - coercedType: coerceFunc: finalType: - "(types.coercedTo ${coercedType} ${coerceFunc} ${finalType})"; - either = val1: val2: "(types.either ${val1} ${val2})"; - loaOf = type: "(types.loaOf ${type})"; - oneOf = of: "(types.oneOf [${lib.concatStringsSep " " (map mapType of)}])"; - }; - - hasTypeMapping = - def: - (def ? oneOf && lib.any hasTypeMapping def.oneOf) - || ( - def ? type - && elem def.type [ - "string" - "integer" - "boolean" - "number" - "any" - ] - ); - - mergeValuesByKey = mergeKey: ''(mergeValuesByKey "${mergeKey}")''; - - mapType = - def: - if def ? oneOf then - types.oneOf (lib.filter hasTypeMapping def.oneOf) - else if def ? type then - if def.type == "string" then - if hasAttr "format" def && def.format == "int-or-string" then - types.either types.int types.str - else - types.str - else if def.type == "integer" then - types.int - else if def.type == "number" then - types.either types.int types.float - else if def.type == "boolean" then - types.bool - else if def.type == "object" then - types.attrs - else if def.type == "array" then - types.listOf (mapType def.items) - else if def.type == "any" then - types.unspecified - else - throw "type ${def.type} not supported" - else - throw "unknown definition"; - - submoduleOf = _definitions: ref: ''(submoduleOf "${ref}")''; - - globalSubmoduleOf = _def: ref: ''(globalSubmoduleOf "${ref}")''; - - submoduleForDefinition = - ref: name: kind: group: version: - ''(submoduleForDefinition "${ref}" "${name}" "${kind}" "${group}" "${version}")''; - - coerceAttrsOfSubmodulesToListByKey = - ref: attrMergeKey: listMergeKeys: - ''(coerceAttrsOfSubmodulesToListByKey "${ref}" "${attrMergeKey}" [${ - concatStringsSep " " (map (key: "\"${toString key}\"") listMergeKeys) - }])''; - - attrsToList = "attrsToList"; - - refDefinition = attr: head (tail (tail (splitString "/" attr."$ref"))); + parts = (import ./walk.nix { inherit lib; }).walk (import ./backend-text.nix { inherit lib; }) { + inherit + schema + specialMapKeys + skipCoerceToList + definitionsOverlay + ; }; - - compareVersions = - ver1: ver2: - let - getVersion = substring 1 10; - splitVersion = v: builtins.splitVersion (getVersion v); - isAlpha = v: elem "alpha" (splitVersion v); - patchVersion = - v: - if isAlpha v then - "" - else if length (splitVersion v) == 1 then - "${getVersion v}prod" - else - getVersion v; - - v1 = patchVersion ver1; - v2 = patchVersion ver2; - in - builtins.compareVersions v1 v2; - - genDefinitions = - definitions: - with gen; - mapAttrs ( - _name: definition: - # if $ref is in definition it means it's an alias of other definition - if hasAttr "$ref" definition then - definitions."${refDefinition definition}" - else if !(hasAttr "properties" definition) then - { } - # in other case it's an actual definition - else - { - options = mapAttrs ( - propName: property: - let - isRequired = elem propName (definition.required or [ ]); - requiredOrNot = type: if isRequired then type else types.nullOr type; - optionProperties = - # if $ref is in property it references other definition, - # but if other definition does not have properties, then just take it's type - if hasAttr "$ref" property then - # Handle our special "#/global" ref prefix, used exclusively to get typed - # metadata from CRDs. - # See: crd2jsonschema.py - if hasPrefix "#/global" property."$ref" then - { - type = requiredOrNot (globalSubmoduleOf definitions (refDefinition property)); - } - else if hasTypeMapping definitions.${refDefinition property} then - { - type = requiredOrNot (mapType definitions.${refDefinition property}); - } - else - { - type = - if (refDefinition property) == _name then - types.unspecified # do not allow self-referential values - else - requiredOrNot (submoduleOf definitions (refDefinition property)); - } - # if property has an array type - else if property.type == "array" then - # if reference is in items it can reference other type of another - # definition - if hasAttr "$ref" property.items then - # if it is a reference to simple type - if hasTypeMapping definitions.${refDefinition property.items} then - { - type = requiredOrNot (types.listOf (mapType definitions.${refDefinition property.items})); - } - # if a reference is to complex type - else - # make it an attribute set of submodules if only x-kubernetes-patch-merge-key is present, or - # x-kubernetes-patch-merge-key == x-kubernetes-list-map-keys. - if - (hasAttr "x-kubernetes-patch-merge-key" property) - && ( - !(hasAttr "x-kubernetes-list-map-keys" property) - || (property."x-kubernetes-list-map-keys" == [ property."x-kubernetes-patch-merge-key" ]) - ) - then - let - mergeKey = property."x-kubernetes-patch-merge-key"; - in - { - type = requiredOrNot ( - coerceAttrsOfSubmodulesToListByKey (refDefinition property.items) mergeKey [ ] - ); - apply = attrsToList; - } - # in other case it's a simple list - else - # make it an attribute set of submodules if only x-kubernetes-patch-merge-key is present, or - # x-kubernetes-patch-merge-key == x-kubernetes-list-map-keys. - if - hasAttr "properties" definitions.${refDefinition property.items} - && hasAttr "name" definitions.${refDefinition property.items}.properties - && !(hasAttr _name skipCoerceToList && any (x: x == propName) skipCoerceToList.${_name}) - then - let - mergeKey = "name"; - in - { - type = requiredOrNot ( - coerceAttrsOfSubmodulesToListByKey (refDefinition property.items) mergeKey ( - if hasAttr _name specialMapKeys && hasAttr propName specialMapKeys.${_name} then - specialMapKeys.${_name}.${propName} - else if hasAttr "x-kubernetes-list-map-keys" property then - property."x-kubernetes-list-map-keys" - else - [ ] - ) - ); - apply = attrsToList; - } - else - { - type = - if (refDefinition property.items) == _name then - types.unspecified # do not allow self-referential values - else - requiredOrNot (types.listOf (submoduleOf definitions (refDefinition property.items))); - } - # in other case it only references a simple type - else - { - type = requiredOrNot (types.listOf (mapType property.items)); - } - else if property.type == "object" && hasAttr "additionalProperties" property then - # if it is a reference to simple type - if - ( - hasAttr "$ref" property.additionalProperties - && hasTypeMapping definitions.${refDefinition property.additionalProperties} - ) - then - { - type = requiredOrNot ( - types.attrsOf (mapType definitions.${refDefinition property.additionalProperties}) - ); - } - else if hasAttr "$ref" property.additionalProperties then - { - type = requiredOrNot types.attrs; - } - # if is an array - else if property.additionalProperties.type == "array" then - { - type = requiredOrNot (types.loaOf (mapType property.additionalProperties.items)); - } - else - { - type = requiredOrNot (types.attrsOf (mapType property.additionalProperties)); - } - # just a simple property - else - { - type = requiredOrNot (mapType property); - }; - in - mkOption ( - { - description = property.description or ""; - } - // optionProperties - ) - ) definition.properties; - config = - let - optionalProps = filterAttrs ( - propName: _property: !(elem propName (definition.required or [ ])) - ) definition.properties; - in - mapAttrs (_name: _property: mkOverride 1002 null) optionalProps; - } - ) definitions; - - definitions = genDefinitions (applyOverlay definitionsOverlay schema.definitions); - resourceTypes = schema.roots; - - resourceTypesByKind = zipAttrs ( - mapAttrsToList (_name: resourceType: { - ${resourceType.kind} = resourceType; - }) resourceTypes - ); - - resourcesTypesByKindSortByVersion = mapAttrs ( - _kind: resourceTypes: - reverseList (sort (r1: r2: compareVersions r1.version r2.version > 0) resourceTypes) - ) resourceTypesByKind; - - latestResourceTypesByKind = mapAttrs (_kind: last) resourcesTypesByKindSortByVersion; - - namespacedResourceTypes = filterAttrs (_: type: type.namespaced) resourceTypes; - - genResourceOptions = - resource: - with gen; - let - submoduleForDefinition' = - definition: - submoduleForDefinition definition.ref definition.name definition.kind definition.group - definition.version; - in - mkOption { - inherit (resource) description; - type = types.attrsOf (submoduleForDefinition' resource); - default = { }; - }; + inherit (parts) + definitions + resourceTypes + latestResourceTypesByKind + namespacedResourceTypes + genResourceOptions + ; generated = '' # This file was generated with nixidy resource generator, do not edit. diff --git a/pkgs/generators/module.nix b/pkgs/generators/module.nix new file mode 100644 index 0000000..ba8fe96 --- /dev/null +++ b/pkgs/generators/module.nix @@ -0,0 +1,89 @@ +# Native module builder for CRDs. +# +# Value-backend assembler: drives the shared schema walk (./walk.nix) with the +# value backend (./backend-value.nix) to produce the resource definitions as a +# Nix *value* — a `{ lib, options, config, ... }: { ... }` module — instead of +# the source file ./generator.nix builds. The result slots straight into +# `nixidy.applicationImports` (which already accepts `functionTo attrs`), so +# there is no generated file and no import-from-derivation: only the Python +# crd2jsonschema parse remains (shared via ./default.nix's `crdSchema`). +{ + name ? "crd", + lib, + schema, + specialMapKeys ? { }, + skipCoerceToList ? { }, + definitionsOverlay ? f: p: p, +}: +# The resource module value. +{ + lib, + options, + config, + ... +}: +with lib; +let + # Shared walk, driven by the value backend. `rt` (the runtime helpers) closes + # over the generated `definitions`, which is itself produced by the walk — + # the mutual recursion is lazy-safe (submodule bodies force `definitions` + # only when an option is evaluated). + parts = (import ./walk.nix { inherit lib; }).walk b { + inherit + schema + specialMapKeys + skipCoerceToList + definitionsOverlay + ; + }; + inherit (parts) + definitions + resourceTypes + latestResourceTypesByKind + namespacedResourceTypes + genResourceOptions + ; + + rt = import ./runtime.nix { inherit lib config definitions; }; + b = import ./backend-value.nix { inherit lib rt; }; +in +{ + # options.resources... plus the latest-version + # aliases. + options.resources = + (foldl' recursiveUpdate { } ( + mapAttrsToList ( + _: r: setAttrByPath [ r.group r.version r.kind ] (genResourceOptions r) + ) resourceTypes + )) + // (mapAttrs' (_: r: nameValuePair r.attrName (genResourceOptions r)) latestResourceTypesByKind); + + config = { + # expose resource definitions + inherit definitions; + + # register resource types + types = mapAttrsToList (_: r: { + inherit (r) + name + group + version + kind + attrName + ; + }) resourceTypes; + + resources = foldl' recursiveUpdate { } ( + mapAttrsToList ( + _: r: + setAttrByPath [ r.group r.version r.kind ] (mkAliasDefinitions options.resources.${r.attrName}) + ) latestResourceTypesByKind + ); + + # make all namespaced resources default to the application's namespace + defaults = mapAttrsToList (_: r: { + inherit (r) group version kind; + default.metadata.namespace = mkDefault config.namespace; + }) namespacedResourceTypes; + }; +} diff --git a/pkgs/generators/runtime.nix b/pkgs/generators/runtime.nix new file mode 100644 index 0000000..29fcc2c --- /dev/null +++ b/pkgs/generators/runtime.nix @@ -0,0 +1,182 @@ +# Runtime helpers for generated resource modules. +# +# These are the exact helper definitions that the text generator +# (./generator.nix) inlines, as source, into the `let` block of every +# generated `.nix` file. The native module builder (./module.nix) imports +# them as real values instead, so it can produce a module *value* directly +# without the generate-source-to-file-then-`import` round-trip. +# +# `definitions` is the generated definition set (`{ = { options; config; }; }`) +# and is threaded back in lazily — `submoduleOf` and friends only force it +# when an option is actually evaluated, so the mutual recursion between the +# helpers and `definitions` is fine. +{ + lib, + config, + definitions, +}: +with lib; +rec { + hasAttrNotNull = attr: set: hasAttr attr set && set.${attr} != null; + + attrsToList = + values: + if values != null then + sort ( + a: b: + if (hasAttrNotNull "_priority" a && hasAttrNotNull "_priority" b) then + a._priority < b._priority + else + false + ) (mapAttrsToList (_n: v: v) values) + else + values; + + getDefaults = + resource: group: version: kind: + catAttrs "default" ( + filter ( + default: + (default.resource == null || default.resource == resource) + && (default.group == null || default.group == group) + && (default.version == null || default.version == version) + && (default.kind == null || default.kind == kind) + ) config.defaults + ); + + types = lib.types // rec { + str = mkOptionType { + name = "str"; + description = "string"; + check = isString; + merge = mergeEqualOption; + }; + + # Either value of type `finalType` or `coercedType`, the latter is + # converted to `finalType` using `coerceFunc`. + coercedTo = + coercedType: coerceFunc: finalType: + mkOptionType rec { + inherit (finalType) getSubOptions getSubModules; + + name = "coercedTo"; + description = "${finalType.description} or ${coercedType.description}"; + check = x: finalType.check x || coercedType.check x; + merge = + loc: defs: + let + coerceVal = + val: + if finalType.check val then + val + else + let + coerced = coerceFunc val; + in + assert finalType.check coerced; + coerced; + in + finalType.merge loc (map (def: def // { value = coerceVal def.value; }) defs); + substSubModules = m: coercedTo coercedType coerceFunc (finalType.substSubModules m); + typeMerge = _t1: _t2: null; + functor = (defaultFunctor name) // { + wrapped = finalType; + }; + }; + }; + + mkOptionDefault = mkOverride 1001; + + mergeValuesByKey = + attrMergeKey: listMergeKeys: values: + listToAttrs ( + imap0 ( + i: value: + nameValuePair ( + if hasAttr attrMergeKey value then + if isAttrs value.${attrMergeKey} then + toString value.${attrMergeKey}.content + else + (toString value.${attrMergeKey}) + else + # generate merge key for list elements if it's not present + "__kubenix_list_merge_key_" + + (concatStringsSep "" ( + map ( + key: if isAttrs value.${key} then toString value.${key}.content else (toString value.${key}) + ) listMergeKeys + )) + ) (value // { _priority = i; }) + ) values + ); + + submoduleOf = + ref: + types.submodule (_: { + options = definitions.${ref}.options or { }; + config = definitions.${ref}.config or { }; + }); + + globalSubmoduleOf = + ref: + types.submodule (_: { + options = config.definitions.${ref}.options or { }; + config = config.definitions.${ref}.config or { }; + }); + + submoduleWithMergeOf = + ref: mergeKey: + types.submodule ( + { name, ... }: + let + convertName = + name: if definitions.${ref}.options.${mergeKey}.type == types.int then toInt name else name; + in + { + options = definitions.${ref}.options // { + # position in original array + _priority = mkOption { + type = types.nullOr types.int; + default = null; + internal = true; + }; + }; + config = definitions.${ref}.config // { + ${mergeKey} = mkOverride 1002 ( + # use name as mergeKey only if it is not coming from mergeValuesByKey + if (!hasPrefix "__kubenix_list_merge_key_" name) then convertName name else null + ); + }; + } + ); + + submoduleForDefinition = + ref: resource: kind: group: version: + let + apiVersion = if group == "core" then version else "${group}/${version}"; + in + types.submodule ( + { name, ... }: + { + inherit (definitions.${ref}) options; + + imports = getDefaults resource group version kind; + config = mkMerge [ + definitions.${ref}.config + { + kind = mkOptionDefault kind; + apiVersion = mkOptionDefault apiVersion; + + # metdata.name cannot use option default, due deep config + metadata.name = mkOptionDefault name; + } + ]; + } + ); + + coerceAttrsOfSubmodulesToListByKey = + ref: attrMergeKey: listMergeKeys: + (types.coercedTo (types.listOf (submoduleOf ref)) (mergeValuesByKey attrMergeKey listMergeKeys) ( + types.attrsOf (submoduleWithMergeOf ref attrMergeKey) + )); +} diff --git a/pkgs/generators/walk.nix b/pkgs/generators/walk.nix new file mode 100644 index 0000000..c9780be --- /dev/null +++ b/pkgs/generators/walk.nix @@ -0,0 +1,244 @@ +# Backend-agnostic CRD schema walk. +# +# Turns a crd2jsonschema schema into the structured pieces both generator +# backends need: the per-definition option set, the resource roots, and the +# version/kind bookkeeping. Every option and type is constructed through the +# supplied `backend` (`b`), so the SAME walk drives both: +# - the text backend (./backend-text.nix) → Nix source, assembled by +# ./generator.nix into a committable `.nix` file, and +# - the value backend (./backend-value.nix) → live module values, assembled +# by ./module.nix into a `{ lib, options, config, ... }: { ... }` module. +# +# Keeping this walk single-sourced is the point: the type/coercion branch logic +# (the part most likely to drift) lives in exactly one place. +{ lib }: +with lib; +let + applyOverlay = + overlay: attr: + let + f = _final: attr; + in + fix (extends overlay f); + + refDefinition = attr: head (tail (tail (splitString "/" attr."$ref"))); + + hasTypeMapping = + def: + (def ? oneOf && any hasTypeMapping def.oneOf) + || ( + def ? type + && elem def.type [ + "string" + "integer" + "boolean" + "number" + "any" + ] + ); + + compareVersions = + ver1: ver2: + let + getVersion = substring 1 10; + splitVersion = v: builtins.splitVersion (getVersion v); + isAlpha = v: elem "alpha" (splitVersion v); + patchVersion = + v: + if isAlpha v then + "" + else if length (splitVersion v) == 1 then + "${getVersion v}prod" + else + getVersion v; + in + builtins.compareVersions (patchVersion ver1) (patchVersion ver2); +in +{ + walk = + b: + { + schema, + skipCoerceToList ? { }, + specialMapKeys ? { }, + definitionsOverlay ? f: p: p, + }: + let + schemaDefs = applyOverlay definitionsOverlay schema.definitions; + resourceTypes = schema.roots; + + resourceTypesByKind = zipAttrs ( + mapAttrsToList (_name: rt: { + ${rt.kind} = rt; + }) resourceTypes + ); + resourcesTypesByKindSortByVersion = mapAttrs ( + _kind: rts: reverseList (sort (r1: r2: compareVersions r1.version r2.version > 0) rts) + ) resourceTypesByKind; + latestResourceTypesByKind = mapAttrs (_kind: last) resourcesTypesByKindSortByVersion; + namespacedResourceTypes = filterAttrs (_: type: type.namespaced) resourceTypes; + + mapType = + def: + if def ? oneOf then + b.types.oneOf (map mapType (filter hasTypeMapping def.oneOf)) + else if def ? type then + if def.type == "string" then + if def ? format && def.format == "int-or-string" then + b.types.either b.types.int b.types.str + else + b.types.str + else if def.type == "integer" then + b.types.int + else if def.type == "number" then + b.types.either b.types.int b.types.float + else if def.type == "boolean" then + b.types.bool + else if def.type == "object" then + b.types.attrs + else if def.type == "array" then + b.types.listOf (mapType def.items) + else if def.type == "any" then + b.types.unspecified + else + throw "type ${def.type} not supported" + else + throw "unknown definition"; + + definitions = mapAttrs ( + _name: definition: + # an alias of another definition + if definition ? "$ref" then + schemaDefs.${refDefinition definition} + else if !(definition ? properties) then + { } + else + { + options = mapAttrs ( + propName: property: + let + isRequired = elem propName (definition.required or [ ]); + requiredOrNot = type: if isRequired then type else b.types.nullOr type; + optionProperties = + if property ? "$ref" then + # Our special "#/global" ref prefix gets typed metadata from + # the shared (config-level) definition set. See crd2jsonschema.py. + if hasPrefix "#/global" property."$ref" then + { type = requiredOrNot (b.globalSubmoduleOf (refDefinition property)); } + else if hasTypeMapping schemaDefs.${refDefinition property} then + { type = requiredOrNot (mapType schemaDefs.${refDefinition property}); } + else + { + type = + if (refDefinition property) == _name then + b.types.unspecified # do not allow self-referential values + else + requiredOrNot (b.submoduleOf (refDefinition property)); + } + else if property.type == "array" then + if property.items ? "$ref" then + if hasTypeMapping schemaDefs.${refDefinition property.items} then + { type = requiredOrNot (b.types.listOf (mapType schemaDefs.${refDefinition property.items})); } + # attrset of submodules merged by x-kubernetes-patch-merge-key + else if + (property ? "x-kubernetes-patch-merge-key") + && ( + !(property ? "x-kubernetes-list-map-keys") + || (property."x-kubernetes-list-map-keys" == [ property."x-kubernetes-patch-merge-key" ]) + ) + then + let + mergeKey = property."x-kubernetes-patch-merge-key"; + in + { + type = requiredOrNot ( + b.coerceAttrsOfSubmodulesToListByKey (refDefinition property.items) mergeKey [ ] + ); + apply = b.attrsToList; + } + # attrset of submodules merged by "name" + else if + schemaDefs.${refDefinition property.items} ? properties + && schemaDefs.${refDefinition property.items}.properties ? name + && !(skipCoerceToList ? ${_name} && any (x: x == propName) skipCoerceToList.${_name}) + then + { + type = requiredOrNot ( + b.coerceAttrsOfSubmodulesToListByKey (refDefinition property.items) "name" ( + if specialMapKeys ? ${_name} && specialMapKeys.${_name} ? ${propName} then + specialMapKeys.${_name}.${propName} + else + property."x-kubernetes-list-map-keys" or [ ] + ) + ); + apply = b.attrsToList; + } + else + { + type = + if (refDefinition property.items) == _name then + b.types.unspecified # do not allow self-referential values + else + requiredOrNot (b.types.listOf (b.submoduleOf (refDefinition property.items))); + } + else + { type = requiredOrNot (b.types.listOf (mapType property.items)); } + else if property.type == "object" && property ? additionalProperties then + if + ( + property.additionalProperties ? "$ref" + && hasTypeMapping schemaDefs.${refDefinition property.additionalProperties} + ) + then + { + type = requiredOrNot ( + b.types.attrsOf (mapType schemaDefs.${refDefinition property.additionalProperties}) + ); + } + else if property.additionalProperties ? "$ref" then + { type = requiredOrNot b.types.attrs; } + else if property.additionalProperties.type == "array" then + { type = requiredOrNot (b.types.loaOf (mapType property.additionalProperties.items)); } + else + { type = requiredOrNot (b.types.attrsOf (mapType property.additionalProperties)); } + else + { type = requiredOrNot (mapType property); }; + in + b.mkOption ( + { + description = property.description or ""; + } + // optionProperties + ) + ) definition.properties; + + config = + let + optionalProps = filterAttrs ( + propName: _property: !(elem propName (definition.required or [ ])) + ) definition.properties; + in + mapAttrs (_name: _property: b.mkOverrideNull) optionalProps; + } + ) schemaDefs; + + genResourceOptions = + resource: + b.mkOption { + inherit (resource) description; + type = b.types.attrsOf ( + b.submoduleForDefinition resource.ref resource.name resource.kind resource.group resource.version + ); + default = { }; + }; + in + { + inherit + definitions + resourceTypes + latestResourceTypesByKind + namespacedResourceTypes + genResourceOptions + ; + }; +}