From 4d8dc8bb7dadba30d5b7c1f1aeec91d4e66baa89 Mon Sep 17 00:00:00 2001 From: Grant Frost Date: Fri, 15 May 2026 15:18:28 -0500 Subject: [PATCH] add oke karpenter fast start optimizer sample --- .../.gitignore | 18 + .../OKE_OPTIMIZED_WORKER_IMAGE_GUIDE.md | 783 ++++++++++++++++++ .../README.md | 215 +++++ .../data.tf | 24 + .../iam.tf | 43 + .../image_builder.tf | 126 +++ .../inflate-100-replicas-deployment.yaml | 34 + .../karpenter.tf | 149 ++++ .../locals.tf | 83 ++ .../network.tf | 201 +++++ .../oke.tf | 87 ++ .../outputs.tf | 57 ++ .../packer/.gitignore | 7 + .../packer/README.md | 145 ++++ .../oke-optimized-worker.example.pkrvars.hcl | 37 + .../packer/oke-optimized-worker.pkr.hcl | 222 +++++ .../packer/scripts/build-from-image-ocid.sh | 125 +++ .../packer/scripts/optimize-oke-worker.sh | 94 +++ .../pause-50-replicas-deployment.yaml | 33 + .../image-builder-cloud-init.sh.tftpl | 65 ++ .../karpenter-optimized-user-data.sh.tftpl | 102 +++ .../terraform.tfvars.example | 41 + .../variables.tf | 290 +++++++ .../versions.tf | 51 ++ 24 files changed, 3032 insertions(+) create mode 100644 oke-karpenter-fast-start-optimizer-sample/.gitignore create mode 100644 oke-karpenter-fast-start-optimizer-sample/OKE_OPTIMIZED_WORKER_IMAGE_GUIDE.md create mode 100644 oke-karpenter-fast-start-optimizer-sample/README.md create mode 100644 oke-karpenter-fast-start-optimizer-sample/data.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/iam.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/image_builder.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/inflate-100-replicas-deployment.yaml create mode 100644 oke-karpenter-fast-start-optimizer-sample/karpenter.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/locals.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/network.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/oke.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/outputs.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/packer/.gitignore create mode 100644 oke-karpenter-fast-start-optimizer-sample/packer/README.md create mode 100644 oke-karpenter-fast-start-optimizer-sample/packer/oke-optimized-worker.example.pkrvars.hcl create mode 100644 oke-karpenter-fast-start-optimizer-sample/packer/oke-optimized-worker.pkr.hcl create mode 100644 oke-karpenter-fast-start-optimizer-sample/packer/scripts/build-from-image-ocid.sh create mode 100644 oke-karpenter-fast-start-optimizer-sample/packer/scripts/optimize-oke-worker.sh create mode 100644 oke-karpenter-fast-start-optimizer-sample/pause-50-replicas-deployment.yaml create mode 100644 oke-karpenter-fast-start-optimizer-sample/templates/image-builder-cloud-init.sh.tftpl create mode 100644 oke-karpenter-fast-start-optimizer-sample/templates/karpenter-optimized-user-data.sh.tftpl create mode 100644 oke-karpenter-fast-start-optimizer-sample/terraform.tfvars.example create mode 100644 oke-karpenter-fast-start-optimizer-sample/variables.tf create mode 100644 oke-karpenter-fast-start-optimizer-sample/versions.tf diff --git a/oke-karpenter-fast-start-optimizer-sample/.gitignore b/oke-karpenter-fast-start-optimizer-sample/.gitignore new file mode 100644 index 0000000..112b7d8 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/.gitignore @@ -0,0 +1,18 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +.terraform/ +*.tfstate +*.tfstate.* +*.tfplan +crash.log +crash.*.log +.DS_Store +*.zip +packer_cache/ +local*.pkrvars.hcl +terraform.tfvars +terraform.tfvars.json +*.auto.tfvars +*.auto.tfvars.json +packer 2/ diff --git a/oke-karpenter-fast-start-optimizer-sample/OKE_OPTIMIZED_WORKER_IMAGE_GUIDE.md b/oke-karpenter-fast-start-optimizer-sample/OKE_OPTIMIZED_WORKER_IMAGE_GUIDE.md new file mode 100644 index 0000000..4106fdb --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/OKE_OPTIMIZED_WORKER_IMAGE_GUIDE.md @@ -0,0 +1,783 @@ + + + +# Optimized OKE Oracle Linux Worker Image Guide + +This guide documents the exact implementation used for the `OKEOptimization` cluster to create faster-starting Oracle Linux 8.10 OKE workers launched by Oracle's Karpenter Provider for OCI (KPO). + +The pattern is: + +1. Start from an Oracle-published OKE worker image, not a generic Oracle Linux image. +2. Build a custom image that already contains the packages and container layers that normally delay first boot. +3. Use KPO/Karpenter to launch nodes from that custom image only when pending workload pods require capacity. +4. Use optimized KPO `user_data` to bootstrap the node with one IMDS read and a direct `oke-install.sh` invocation. +5. Measure the node lifecycle with explicit bootstrap markers. + +## Final Environment + +The implementation in this repository uses: + +| Component | Value | +| --- | --- | +| Region | `us-ashburn-1` | +| OKE version | `v1.35.2` | +| KPO chart version | `v1.1.0` | +| Base worker image | `Oracle-Linux-8.10-2026.02.28-0-OKE-1.35.2-1402` | +| Base image OCID | `` | +| Custom image name | `oke-optimization-ol8-oke-1-35-2-fast-start` | +| Custom image OCID | Generated during image build | +| Karpenter NodePool / OCINodeClass | `optimized-ol8` | +| Karpenter worker shape | `VM.Standard.E5.Flex` | +| Karpenter worker size | `4 OCPU`, `16 GB` memory | +| NodePool limits | `cpu: "96"`, `memory: 384Gi` | +| CNI | `OCI_VCN_IP_NATIVE` | +| Secondary VNIC pod IP count | `16` | + +Important: rebuild the custom image whenever the OKE worker image, Kubernetes minor/patch version, region, or required pre-pulled image set changes. + +## Implementation Files + +The current implementation is split across these files: + +| File | Purpose | +| --- | --- | +| `image_builder.tf` | Launches the temporary image-builder VM, waits for the build marker, stops the VM, and creates the custom image. | +| `templates/image-builder-cloud-init.sh.tftpl` | Performs image-time package install, CRI-O image pre-pulls, verification, and cleanup. | +| `karpenter.tf` | Installs KPO, creates the `OCINodeClass`, and creates the `NodePool`. | +| `templates/karpenter-optimized-user-data.sh.tftpl` | Runtime bootstrap wrapper used by KPO-launched nodes. | +| `locals.tf` | Renders KPO user data, shape configs, network config, and custom image tags. | +| `variables.tf` | Defines worker shape, limits, pre-pulled images, image version, and region defaults. | +| `iam.tf` | Creates the KPO workload identity policies and node dynamic group policy for `CLUSTER_JOIN`. | + +## Step 1: Use Karpenter/KPO for Scale From Zero + +Use Karpenter Provider for OCI rather than a fixed-size managed node pool for the scaling test workers. + +Why: + +- Karpenter reacts to unschedulable pods and creates just enough nodes. +- The optimized workers can scale from zero; no idle optimized workers are required before the test. +- It avoids mixing scaling-test workloads with the small managed OKE node pool that hosts the KPO controller. + +Implementation details: + +- The cluster still has one regular OKE managed bootstrap node pool for the KPO controller. +- Optimized application pods are forced onto Karpenter nodes by using: + - node selector: `oke-optimization.oracle.com/profile=fast-start` + - toleration: `oke-optimization.oracle.com/scaling-test=true:NoSchedule` +- The Karpenter NodePool applies the matching label and taint: + - label: `oke-optimization.oracle.com/profile=fast-start` + - taint: `oke-optimization.oracle.com/scaling-test=true:NoSchedule` + +This makes the test a true scale-from-zero test. If 100 pods are created, they cannot land on the regular OKE bootstrap node. + +## Step 2: Start From an OKE-Ready Oracle Linux Image + +Use the Oracle-published OKE worker image for the exact Kubernetes version. + +For this implementation: + +```text +Oracle-Linux-8.10-2026.02.28-0-OKE-1.35.2-1402 +``` + +Why: + +- OKE worker images already contain OKE-specific bootstrap assets such as `/etc/oke/oke-install.sh`. +- KPO expects an OKE-compatible image when using `imageType = "OKEImage"`. +- Starting from a generic Oracle Linux image would require recreating OKE bootstrap behavior and would be fragile. + +The custom image must be rebuilt for each OKE worker image version. Do not reuse a custom image built for an older OKE Kubernetes release unless it has been explicitly validated against the target cluster version. + +## Step 3: Build a Custom Image With a Temporary Builder VM + +Terraform launches a temporary OCI compute instance from the base OKE image. + +Builder settings used here: + +| Setting | Value | +| --- | --- | +| Shape | `VM.Standard.E5.Flex` | +| OCPUs | `4` | +| Memory | `16 GB` | +| Boot volume | same worker boot volume size as the Karpenter workers | +| Network | dedicated public image-builder subnet | +| SSH | configured project public key | + +Why: + +- The builder VM runs the same OS and OKE worker image that Karpenter nodes will use. +- Any files, packages, and CRI-O image layers written during the build are captured into the custom image. +- Terraform can verify the build over SSH before creating the custom image. + +The builder workflow is: + +1. Launch builder from the base OKE worker image. +2. Run `templates/image-builder-cloud-init.sh.tftpl`. +3. Wait for `/var/lib/oke-optimization/image-build-complete`. +4. Soft-stop the builder VM with OCI CLI. +5. Create `oci_core_image.optimized_oke_worker` from the stopped instance. +6. Tag the image with the base image and optimization metadata. + +## Step 4: Bake Package Downloads Into the Image + +The builder installs: + +```bash +dnf -y install oraclelinux-developer-release-el8 +dnf -y install jq curl python36-oci-cli +``` + +Why: + +- Internal testing identified node-start downloads as one of the largest delays in cold boot. +- Installing these packages during every Karpenter node boot depends on repository availability, mirror latency, and node network startup timing. +- `jq` is required by the optimized runtime bootstrap script to parse the single IMDS JSON response. +- `curl` is required by the optimized bootstrap path. +- `python36-oci-cli` was included because the original image-bake path included it and because some OKE/customer bootstrap variants use OCI CLI during node setup. The final optimized runtime script does not use OCI CLI for reserved public IP assignment. + +Issue solved: + +- Removes package manager and external repository latency from every Karpenter scale-up node. +- Avoids repeated package downloads during large scale-out events. + +What helped: + +- Baking downloads into the image is one of the most important optimizations. The initial analysis estimated this class of delay at roughly 45-90 seconds per node in the original cold-start path. + +## Step 5: Pre-Pull Core OKE Images at Image Build Time + +The builder starts CRI-O only during the image build: + +```bash +systemctl start crio +``` + +Then it pre-pulls these Ashburn-region OKE images into CRI-O storage: + +```text +iad.ocir.io/axoxdievda5j/oke-public-pause@sha256:6e287874898efea60adff9dc1faaee355740b192763f963fe66cf755d995cf32 +iad.ocir.io/id9y6mi8tcky/oke-public-vcn-native-ip-cni-plugin@sha256:916f9953b84f2788b410c45ba766901e99d9c345d37e2fc35a16c312f27e34f0 +iad.ocir.io/id9y6mi8tcky/oke-public-kube-proxy@sha256:175559244baa3cbee68adfc1357c18871406c7a2d49dd487d2f04e111e7931a0 +``` + +Why: + +- The original cloud-init pre-pulled `pause`, the VCN-native CNI plugin, and `kube-proxy`. +- Pulling those images during runtime adds registry/network latency to every new node. +- During a large scale-out event, many nodes pulling the same images at once can amplify registry and network delay. +- Digest-pinned images are deterministic and avoid unexpected `latest` tag changes. + +Issue solved: + +- Removes core OKE image pull latency from the critical path. +- Reduces registry fan-out during scale tests. + +What helped: + +- This helped, but the implementation that helped was image-time pre-pull, not runtime pre-pull. Later pod events showed image pulls were either already present or completed in a few hundred milliseconds, so image pull was no longer the scaling bottleneck. + +## Step 6: Verify Pre-Pulled Images Before Capturing the Image + +The image build verifies each configured image exists in CRI-O storage: + +```bash +crictl images --digests | grep -F "$image_repo" >/dev/null +``` + +Why: + +- A custom image that silently misses one of the expected layers is hard to diagnose later. +- Verification fails the image build visibly instead of producing a partially optimized image. + +Issue solved: + +- Prevents false-positive image builds. +- Ensures the later Karpenter scale test is testing the intended optimization. + +## Step 7: Clean Only Transient Builder State + +The builder cleans transient data before image capture: + +```bash +dnf clean all +rm -rf /var/cache/dnf/* /var/tmp/* /tmp/* +rm -rf /var/lib/cloud/instances/* /var/lib/cloud/instance +rm -f /root/.bash_history /home/opc/.bash_history +find /var/log -type f ! -name "oke-optimized-image-build.log" -exec truncate -s 0 {} \; || true +truncate -s 0 /etc/machine-id || true +``` + +Why: + +- Cloud-init instance state must not be baked into a reusable image. +- Machine identity must be regenerated per instance. +- DNF cache and temporary files should not inflate the image. +- The OKE bootstrap assets and CRI-O image storage must be preserved. + +Issue solved: + +- Avoids cloned machine identity and stale cloud-init state. +- Keeps the image reusable while preserving the pre-pulled image layers. + +Important: + +- Do not delete `/etc/oke`. +- Do not delete CRI-O container storage after pre-pulling images. + +## Step 8: Tag the Custom Image + +The custom image is tagged with: + +```hcl +k8s_version = "v1.35.2" +base_image_ocid = "" +base_image_name = "Oracle-Linux-8.10-2026.02.28-0-OKE-1.35.2-1402" +optimization_profile = "fast-start" +``` + +Why: + +- Teams need to know exactly which OKE worker image a custom image was built from. +- Karpenter worker images are tightly coupled to OKE/Kubernetes versions. +- Tags make image provenance clear during later audits and rebuilds. + +Issue solved: + +- Prevents accidental use of stale or mismatched worker images. + +## Golden Image Drift and Staleness Risks + +If this image build process is moved to Packer, the same optimization pattern still applies. Packer replaces the Terraform temporary-builder workflow, but the output is still a golden image. That means all baked packages and pre-pulled container layers are frozen at image-build time. + +Items frozen into the image include: + +- OS packages installed with `dnf` +- `oraclelinux-developer-release-el8` +- `jq` +- `curl` +- `python36-oci-cli` +- CRI-O cached image layers +- any application images teams choose to pre-pull + +Why this matters: + +- The speed improvement comes from not downloading those items during every node boot. +- The risk is that the image can drift behind Oracle's latest OKE worker image, OS package fixes, CRI-O fixes, OCI CLI fixes, or newer Kubernetes infrastructure image digests. +- A stale image may continue to boot successfully while silently missing security fixes or compatibility updates. + +Pre-pulled image behavior: + +- Pre-pulling an image into CRI-O does not permanently force Kubernetes to use that layer if the live DaemonSet asks for a different image reference. +- If the requested image digest is different, CRI-O must pull the different digest. +- If the requested image uses a mutable tag such as `latest`, cached behavior can become ambiguous and may use stale content depending on pull policy and runtime behavior. +- For that reason, this implementation uses region-correct digest-pinned images instead of mutable `latest` tags. + +Required operating model: + +1. Treat the optimized worker image as versioned per OKE/Kubernetes version. +2. Rebuild from Oracle's newest certified OKE worker image for the target Kubernetes version. +3. Rebuild when any pre-pulled image digest changes. +4. Rebuild on a regular security cadence, such as monthly, even if the OKE version does not change. +5. Promote a new image only after a node-join smoke test and Karpenter scale test. +6. Tag each image with build timestamp, base image OCID, base image name, Kubernetes version, optimization profile, and the pre-pulled image digest set. +7. Retire old optimized images after the replacement has passed validation. + +Important package guidance: + +- Prefer rebuilding from Oracle's latest OKE worker image over heavily mutating an older worker image. +- Do not run a broad, untested `dnf update -y` on top of an OKE worker image without validation. +- OKE worker images are curated artifacts; replacing core packages outside Oracle's image cadence can introduce compatibility risk. +- If a security team requires package updates between Oracle OKE image releases, validate the updated image with a full node join, CNI, kube-proxy, DaemonSet, and workload scheduling test before promotion. + +## Step 9: Configure KPO to Use the Custom Image + +The `OCINodeClass` uses the custom image in the boot volume image config: + +```yaml +volumeConfig: + bootVolumeConfig: + imageConfig: + imageType: OKEImage + imageId: +``` + +Why: + +- This tells KPO to launch OKE-compatible workers from the optimized image. +- Karpenter still owns instance creation, but the boot disk comes from the prebuilt fast-start image. + +Issue solved: + +- Ensures every Karpenter-provisioned worker starts with the baked packages and CRI-O image layers. + +## Step 10: Use Full KPO `metadata.user_data` for Runtime Bootstrap + +The final bootstrap script is rendered into: + +```yaml +OCINodeClass.spec.metadata.user_data +``` + +It is not implemented as a small pre-bootstrap hook. The full user-data wrapper owns the optimized bootstrap path and then invokes Oracle's OKE installer. + +Why: + +- The node still needs cluster-specific data at runtime: API server endpoint, cluster CA, DNS, and node identity. +- Those values cannot be fully baked into the image. +- KPO-provided user data is the right place to combine runtime metadata with the prebuilt image. + +Issue solved: + +- Keeps the image reusable across node instances while avoiding repeated metadata/polling behavior. + +## Step 11: Fetch IMDS Once + +The runtime bootstrap performs one IMDS call: + +```bash +INSTANCE_JSON=$(curl -sfL -m 5 --retry 3 \ + -H "Authorization: Bearer Oracle" \ + "http://169.254.169.254/opc/v2/instance") +``` + +Then it extracts values locally: + +```bash +APISERVER=$(echo "$INSTANCE_JSON" | jq -er '.metadata.apiserver_host') +CA_CERT=$(echo "$INSTANCE_JSON" | jq -er '.metadata.cluster_ca_cert') +DNS=$(echo "$INSTANCE_JSON" | jq -r '.metadata.cluster_dns // .metadata.kubedns_svc_ip // ""') +``` + +Why: + +- Internal analysis found that multiple sequential metadata calls and installer polling added measurable startup delay. +- A single metadata document fetch removes repeated IMDS round trips. +- Local `jq` extraction is fast and deterministic. + +Issue solved: + +- Avoids repeated IMDS curl calls during the critical boot path. +- Ensures the bootstrap script fails clearly if required metadata is missing. + +What helped: + +- This was one of the direct recommendations from the optimization notes and it remains in the final implementation. It is part of the faster and more predictable bootstrap path. + +## Step 12: Call `/etc/oke/oke-install.sh` With Explicit Values + +The implementation does not modify Oracle's `/etc/oke/oke-install.sh` file. + +The enhancement is in how the script is invoked: + +```bash +bash /etc/oke/oke-install.sh \ + --apiserver-endpoint "$APISERVER" \ + --kubelet-ca-cert "$CA_CERT" \ + --cluster-dns "$DNS" \ + --kubelet-extra-args "$KUBELET_ARGS" +``` + +Why: + +- Internal analysis identified that passing explicit values avoids internal metadata lookup and polling loops in the installer path. +- Keeping Oracle's script unmodified reduces maintenance risk when OKE worker images change. +- The custom wrapper is easier to diff and validate than a patched Oracle bootstrap script. + +Issue solved: + +- Removes avoidable discovery/polling time from the bootstrap path. +- Preserves vendor-provided OKE bootstrap behavior. + +What helped: + +- This materially helped. It is one of the key runtime optimizations that stayed in the final implementation. + +## Step 13: Set Kubelet Reservations Explicitly + +The runtime bootstrap passes: + +```bash +KUBELET_ARGS="\ +--kube-reserved=cpu=70m,memory=1Gi \ +--system-reserved=cpu=100m,memory=256Mi \ +--eviction-hard=memory.available<500Mi" +``` + +Why: + +- The optimization notes called out that default reservations on smaller nodes were too low. +- Undersized reservations can lead to node pressure, instability, and delayed readiness under load. +- The current demo workers are modestly sized at 4 OCPU and 16 GB memory, so explicit reservations are appropriate. + +Issue solved: + +- Improves node stability during bootstrap and scale tests. +- Reduces risk that DaemonSets, kubelet, or system processes compete too aggressively with workload pods. + +What helped: + +- This is primarily a stability improvement, not the largest raw boot-speed improvement. It should be treated as a best-practice baseline for smaller worker nodes. + +## Step 14: Do Not Start CRI-O or Pull Images at Runtime + +The attached cloud-init started background pulls during bootstrap. That was tested, but the final implementation removed runtime CRI-O start and runtime pull loops. + +Final runtime behavior: + +- Do not run `systemctl start crio` before `oke-install.sh`. +- Do not run background `crictl pull` loops before or during `oke-install.sh`. +- Go from IMDS extraction directly to `/etc/oke/oke-install.sh`. + +Why: + +- Starting CRI-O manually during runtime can race with OKE bootstrap's own service ordering. +- Node-side analysis showed CRI-O-related stalls and worse launch-to-register timing when runtime CRI-O start/pulls were present. +- The image-time pre-pull already captures the useful part of the image pull optimization. + +Issue solved: + +- Avoids CRI-O bootstrap races. +- Keeps image pull latency out of the node boot path without destabilizing OKE startup. + +What helped: + +- The concept of pre-pulling helped. +- Runtime background pulls did not help in this environment once the image was pre-baked; they hurt consistency. +- The final winning design is "pre-pull during image build, then leave CRI-O startup to OKE bootstrap at runtime." + +## Step 15: Add Bootstrap Timing Markers + +The runtime bootstrap writes a timing log: + +```text +/var/log/oke-optimized-bootstrap-timing.log +``` + +It also writes marker files under: + +```text +/var/lib/oke-optimization/ +``` + +Events recorded: + +```text +imds-fetch-start +imds-fetch-end +oke-installer-start +oke-installer-end +kubelet-active +node-ready +``` + +Each log line includes: + +- ISO timestamp +- epoch timestamp +- node uptime seconds +- event name + +Why: + +- Karpenter and Kubernetes events show the outside view of provisioning. +- Node-side markers show where time is spent inside the guest OS. +- Future teams can separate OCI provisioning time from node bootstrap time. + +Issue solved: + +- Makes boot regressions measurable. +- Prevents guessing whether the bottleneck is OCI provisioning, OKE install, kubelet activation, image pulls, or scheduling. + +## Step 16: Configure KPO IAM Correctly + +KPO needs workload identity permissions to create and manage OCI resources. + +Implemented KPO workload policies allow the KPO service account to: + +- manage `instance-family` +- manage `volumes` +- manage `volume-attachments` +- manage `virtual-network-family` +- read `instance-images` +- inspect compartments + +Karpenter-launched nodes also need `CLUSTER_JOIN`. + +Implemented node dynamic group: + +```text +ALL {instance.compartment.id = ''} +``` + +Implemented node policy: + +```text +Allow dynamic-group to {CLUSTER_JOIN} +in compartment OKEOptimization +where target.cluster.id = '' +``` + +Why: + +- KPO cannot launch nodes without OCI compute/network permissions. +- Self-managed Karpenter nodes cannot join OKE without `CLUSTER_JOIN`. + +Issue solved: + +- Prevents nodes from launching but failing to register with the cluster. +- Keeps the permissions scoped to the project compartment and cluster. + +## Step 17: Tune NodePool Behavior for the Demo + +The final NodePool settings are: + +```yaml +disruption: + budgets: + - nodes: 100% + consolidateAfter: 30s + consolidationPolicy: WhenEmptyOrUnderutilized +template: + spec: + terminationGracePeriod: 2m +limits: + cpu: "96" + memory: 384Gi +``` + +Why: + +- `consolidateAfter: 30s` makes the demo scale down quickly after workloads are deleted. +- `WhenEmptyOrUnderutilized` allows downsizing when nodes are empty or underutilized. +- `nodes: 100%` allows aggressive disruption for a demo cluster. +- `terminationGracePeriod: 2m` prevents long termination delays during demos. +- The limits were raised from `64/256Gi` to `96/384Gi` after a 100-pod test hit the cap. + +Issue solved: + +- Avoids a repeat of the 100-pod test where 8 nodes ran 96 pods and the final 4 pods remained pending because the NodePool CPU limit was exhausted. + +## Step 18: Validate the Optimized Image and Runtime Path + +After applying Terraform, validate: + +```bash +terraform fmt -recursive +terraform validate +terraform plan +terraform apply +``` + +Verify the custom image: + +```bash +oci compute image get --image-id +``` + +Confirm: + +- lifecycle state is `AVAILABLE` +- image compartment is `OKEOptimization` +- tags include `k8s_version`, `base_image_ocid`, `base_image_name`, and `optimization_profile = fast-start` + +Verify KPO sees the image: + +```bash +kubectl --context OKEOptimization get ocinodeclass optimized-ol8 -o yaml +``` + +Confirm: + +- `Ready=True` +- the resolved image ID matches the custom image OCID + +Verify the NodePool: + +```bash +kubectl --context OKEOptimization get nodepool optimized-ol8 -o yaml +``` + +Confirm: + +- limits are `cpu: "96"` and `memory: 384Gi` +- consolidation is `WhenEmptyOrUnderutilized` +- `consolidateAfter` is `30s` +- taint and label are present + +## Step 19: Run a True Scale Test + +The test workload must include the selector and toleration: + +```yaml +nodeSelector: + oke-optimization.oracle.com/profile: fast-start +tolerations: +- key: oke-optimization.oracle.com/scaling-test + operator: Equal + value: "true" + effect: NoSchedule +``` + +Why: + +- Without the selector, pods may land on the regular OKE bootstrap node. +- Without the toleration, pods cannot land on the tainted Karpenter nodes. + +Example workload sizing used in testing: + +```yaml +replicas: 100 +resources: + requests: + cpu: 500m + memory: 512Mi +``` + +For this demo, limits are optional. Scheduling is driven by requests, not limits. + +## Step 20: Measure the Scale-Up Path + +Use Karpenter and Kubernetes events: + +```bash +kubectl --context OKEOptimization get nodeclaims +kubectl --context OKEOptimization get nodes -l karpenter.sh/nodepool=optimized-ol8 +kubectl --context OKEOptimization get pods -o wide +kubectl --context OKEOptimization logs -n karpenter deploy/karpenter +``` + +Use node-side markers: + +```bash +sudo cat /var/log/oke-optimized-bootstrap-timing.log +sudo ls -l /var/lib/oke-optimization/ +``` + +The important lifecycle segments are: + +| Segment | What it means | +| --- | --- | +| Pod created to Karpenter NodeClaim created | Scheduler/Karpenter reaction time | +| NodeClaim created to launched | KPO and OCI instance launch path | +| Launched to registered | Instance boot plus OKE bootstrap | +| Registered to initialized/Ready | Kubelet, node conditions, and daemon readiness | +| Scheduled to container started | Pod image availability and runtime startup | + +In the latest validated run after removing runtime CRI-O start/pulls: + +| Segment | Observed result | +| --- | --- | +| NodeClaim create to launched | average about 35s, max about 42s | +| Launched to registered | average about 52s, max about 64s | +| Registered to initialized | average about 11s, max about 16s | +| NodeClaim create to ready | average about 98s, max about 107s | +| Scheduled to container started | average about 14s | + +Image pulls were no longer the bottleneck; pod events showed images were already present or pulled in a few hundred milliseconds. + +## Optimization Recommendations: Final Status + +| Recommendation from optimization notes | Final status | Helped? | Notes | +| --- | --- | --- | --- | +| Use Karpenter with custom OKE images | Implemented | Yes | Enables scale from zero using optimized workers. | +| Bake downloads into a custom/golden image | Implemented | Yes | Removed per-node DNF/repository dependency from boot. | +| Install `oraclelinux-developer-release-el8`, `jq`, `curl`, `python36-oci-cli` | Implemented | Yes | `jq` and `curl` are required by final bootstrap; OCI CLI is baked for compatibility with the original dependency set. | +| Fetch metadata with one IMDS request | Implemented | Yes | Runtime bootstrap fetches `/opc/v2/instance` once and parses locally. | +| Pass explicit API server, CA, and DNS to `oke-install.sh` | Implemented | Yes | Avoids extra discovery/polling while preserving Oracle's installer. | +| Pre-pull `pause`, CNI, and `kube-proxy` | Implemented differently | Yes | Done at image build time with Ashburn digest-pinned images, not as runtime pulls. | +| Start background `crictl pull` during bootstrap | Removed | No, not in final form | Runtime pulls caused consistency issues when combined with OKE bootstrap. Image-time pulls kept the benefit. | +| Add kubelet reservations | Implemented | Yes, mostly stability | Helps small-node reliability and pressure behavior. | +| Reserved public IP logic with OCI CLI | Not implemented | No | The notes also recommended removing it as expensive and edge-case. | +| Warm capacity / warm pools | Not implemented | Potential future work | Would reduce perceived provisioning delay but was outside this Terraform design. | +| Pre-pull application images | Not implemented for demo | Depends on app | Teams should add large application images to `prepull_images` if their workloads are image-pull bound. | + +## What Changed in the OKE Installer Path + +No changes were made to `/etc/oke/oke-install.sh` itself. + +The enhancement is a wrapper that: + +1. Fetches metadata once. +2. Extracts API server, CA, and DNS locally. +3. Supplies those values directly to `/etc/oke/oke-install.sh`. +4. Supplies kubelet reservation/eviction args explicitly. +5. Records timing markers around IMDS, installer, kubelet active, and node ready. + +This keeps the Oracle-provided installer intact while removing avoidable discovery work around it. + +## What Actually Improved Boot Speed + +The changes that truly improved or protected boot speed were: + +1. Baking dependency downloads into the image. +2. Baking core OKE image layers into CRI-O storage. +3. Fetching IMDS once instead of making multiple metadata calls. +4. Passing explicit bootstrap values to `oke-install.sh`. +5. Removing runtime CRI-O start and background pulls after testing showed they hurt consistency. + +The changes that mainly improved operability and demo behavior were: + +1. Kubelet reservations. +2. Bootstrap timing markers. +3. NodePool selector/taint isolation. +4. Fast consolidation and 2-minute termination grace. +5. Increased NodePool limits. + +## Prescriptive Checklist for a New Team + +Use this checklist for each new OKE version or region: + +1. Select the Oracle-published OKE worker image for the exact Kubernetes version and region. +2. Launch a temporary builder VM from that image. +3. Install `oraclelinux-developer-release-el8`, `jq`, `curl`, and `python36-oci-cli`. +4. Start CRI-O on the builder only. +5. Pre-pull required OKE infrastructure images by digest into CRI-O storage. +6. Verify the images exist with `crictl images --digests`. +7. Clean DNF cache, cloud-init instance state, logs, temp files, shell history, and machine ID. +8. Do not remove `/etc/oke` or CRI-O image storage. +9. Stop the builder VM. +10. Create a custom OCI image from the stopped builder. +11. Tag the custom image with Kubernetes version, base image OCID, base image name, and optimization profile. +12. Configure KPO `OCINodeClass` to use the custom image as `imageType = "OKEImage"`. +13. Put optimized runtime bootstrap in `OCINodeClass.spec.metadata.user_data`. +14. In runtime bootstrap, fetch IMDS once from `/opc/v2/instance`. +15. Parse API server, cluster CA, and DNS locally with `jq`. +16. Invoke `/etc/oke/oke-install.sh` with explicit endpoint, CA, DNS, and kubelet args. +17. Do not start CRI-O manually at runtime. +18. Do not run runtime background `crictl pull` loops if the image already contains the layers. +19. Add timing markers for IMDS, installer, kubelet active, and node ready. +20. Configure KPO IAM policies and node `CLUSTER_JOIN` dynamic group. +21. Use Karpenter NodePool labels and taints so test workloads only land on optimized nodes. +22. Set NodePool limits high enough for the expected scale test. +23. Validate with Terraform and a real pending-pod scale test. +24. Review Karpenter logs, NodeClaims, Kubernetes events, and node timing markers after each run. + +## Common Failure Modes + +| Symptom | Likely cause | Fix | +| --- | --- | --- | +| Node launches but never joins OKE | Missing `CLUSTER_JOIN` policy for Karpenter nodes | Add compartment-based dynamic group and cluster-scoped `CLUSTER_JOIN` policy. | +| Pods remain pending even though Karpenter works | NodePool CPU/memory limits too low | Raise NodePool limits based on expected nodes and reported Kubernetes CPU capacity. | +| Pods land on the regular OKE node pool | Missing node selector | Add `oke-optimization.oracle.com/profile=fast-start`. | +| Pods cannot land on Karpenter nodes | Missing toleration | Add toleration for `oke-optimization.oracle.com/scaling-test=true:NoSchedule`. | +| Node boot becomes inconsistent | Runtime CRI-O start or runtime image pulls racing OKE bootstrap | Remove runtime CRI-O start/pulls; pre-pull during image build only. | +| Image build succeeds but runtime still pulls core images | Pre-pull verification missing or wrong image region/tag | Use region-correct, digest-pinned images and verify with `crictl images --digests`. | +| Custom image works on one OKE version but not another | Image version mismatch | Rebuild from the OKE worker image for the target Kubernetes version. | + +## Summary + +The optimized worker image is not just a custom image. The speed comes from combining: + +- an OKE-ready base image, +- image-time package baking, +- image-time CRI-O layer pre-pulls, +- a single IMDS metadata fetch, +- direct `oke-install.sh` arguments, +- explicit kubelet reservations, +- no runtime CRI-O interference, +- KPO/Karpenter scale-from-zero behavior, +- correct OCI IAM for KPO and `CLUSTER_JOIN`, +- and repeatable timing validation. + +That combination is what produced the improved scale behavior seen in the demo cluster. diff --git a/oke-karpenter-fast-start-optimizer-sample/README.md b/oke-karpenter-fast-start-optimizer-sample/README.md new file mode 100644 index 0000000..bdcfeaa --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/README.md @@ -0,0 +1,215 @@ + + + +# OKE Karpenter Fast-Start Workers + +Reference implementation for improving Oracle Kubernetes Engine (OKE) scale-out consistency with Oracle's Karpenter Provider for OCI (KPO), optimized Oracle Linux OKE worker images, and Terraform-managed OCI infrastructure. + +This project is intended for engineering validation, demos, and as a starting point for production hardening. It is not a replacement for normal OKE upgrade, security, and image promotion processes. + +## Problem + +Kubernetes burst scaling can be slowed by work that happens repeatedly on every new node: + +- package downloads during node boot, +- image pulls for core Kubernetes/OKE components, +- repeated instance metadata lookups, +- bootstrap logic that rediscover values already available in metadata, +- slow or inconsistent scale-down behavior during demos. + +When many nodes start at once, these delays stack across the cluster and make scale-out timing harder to predict. + +## What This Project Builds + +- An OKE enhanced cluster in OCI. +- A small managed OKE node pool used only for the KPO controller. +- KPO installed by Helm. +- IAM policies and a dynamic group for KPO operation and self-managed node `CLUSTER_JOIN`. +- A custom Oracle Linux OKE worker image with image-time optimizations. +- A Karpenter `OCINodeClass` and `NodePool` that launch optimized workers from zero. +- An optimized runtime bootstrap script that calls `/etc/oke/oke-install.sh` with explicit values. +- A Packer pipeline for building the optimized worker image from a base OKE worker image OCID. +- Example workloads for scale testing. + +## Optimization Pattern + +The implementation intentionally splits work into two phases. + +Image build time: + +- Start from an Oracle-published OKE worker image for the exact Kubernetes version. +- Install required packages such as `oraclelinux-developer-release-el8`, `jq`, `curl`, and `python36-oci-cli`. +- Start CRI-O only during image build. +- Pre-pull digest-pinned OKE infrastructure images into CRI-O storage. +- Verify the expected image layers are present. +- Clean transient package, cloud-init, log, history, and machine identity state. +- Preserve `/etc/oke` and CRI-O image storage. + +Node boot time: + +- Fetch instance metadata once. +- Extract API server, cluster CA, and DNS locally with `jq`. +- Call `/etc/oke/oke-install.sh` directly with explicit bootstrap values. +- Pass kubelet reservation and eviction arguments. +- Let OKE manage CRI-O startup order. +- Record timing markers for later boot analysis. + +## Repository Layout + +```text +. +├── *.tf # Terraform OCI, OKE, IAM, KPO, and Karpenter resources +├── templates/ +│ ├── image-builder-cloud-init.sh.tftpl # Terraform image-builder cloud-init path +│ └── karpenter-optimized-user-data.sh.tftpl +├── packer/ +│ ├── oke-optimized-worker.pkr.hcl # OCI Packer builder +│ ├── oke-optimized-worker.example.pkrvars.hcl +│ ├── scripts/ +│ │ ├── build-from-image-ocid.sh +│ │ └── optimize-oke-worker.sh +│ └── README.md +├── terraform.tfvars.example +├── OKE_OPTIMIZED_WORKER_IMAGE_GUIDE.md +├── inflate-100-replicas-deployment.yaml +└── pause-50-replicas-deployment.yaml +``` + +## Prerequisites + +- OCI tenancy and compartment for the project. +- OCI CLI configured with a profile that can manage OKE, compute, networking, IAM policies, and images. +- Terraform. +- Packer, if using the Packer image build path. +- `kubectl`. +- `helm`. +- SSH key pair for the temporary image-builder VM and worker access. +- Network path that allows the local machine or CI runner to SSH into the temporary image-builder instance. + +## Terraform Quick Start + +Copy the example variables: + +```sh +cp terraform.tfvars.example terraform.tfvars +``` + +Update at minimum: + +```hcl +tenancy_ocid = "REPLACE_WITH_TENANCY_OCID" +parent_compartment_ocid = "REPLACE_WITH_PARENT_COMPARTMENT_OCID" +project_compartment_ocid = "REPLACE_WITH_PROJECT_COMPARTMENT_OCID" +oke_image_compartment_ocid = "REPLACE_WITH_ORACLE_OKE_IMAGE_COMPARTMENT_OCID" +ssh_public_key_path = "~/.ssh/id_rsa.pub" +ssh_private_key_path = "~/.ssh/id_rsa" +image_builder_ssh_cidr = "203.0.113.10/32" +allowed_kubernetes_api_cidrs = ["203.0.113.10/32"] +``` + +Then run: + +```sh +terraform init +terraform fmt -recursive +terraform validate +terraform plan +terraform apply +``` + +## Packer Image Build + +The Packer pipeline can build an optimized image from a base OKE worker image OCID: + +```sh +cd packer +packer init . +./scripts/build-from-image-ocid.sh +``` + +To validate without creating an image: + +```sh +PACKER_VALIDATE_ONLY=true ./scripts/build-from-image-ocid.sh +``` + +For fully portable use, copy and edit: + +```sh +cp oke-optimized-worker.example.pkrvars.hcl local.pkrvars.hcl +packer build -var-file=local.pkrvars.hcl . +``` + +## Karpenter Scale Test + +The test workloads are pinned to Karpenter optimized nodes with: + +```yaml +nodeSelector: + oke-optimization.oracle.com/profile: fast-start +tolerations: +- key: oke-optimization.oracle.com/scaling-test + operator: Equal + value: "true" + effect: NoSchedule +``` + +This prevents test pods from landing on the regular managed OKE node pool. + +Apply a test workload manually: + +```sh +kubectl apply -f inflate-100-replicas-deployment.yaml +``` + +Watch scale-out: + +```sh +kubectl get nodeclaims +kubectl get nodes -l karpenter.sh/nodepool=optimized-ol8 +kubectl get pods -o wide +kubectl logs -n karpenter deploy/karpenter +``` + +## Cleanup + +Delete any sample workloads before destroying the infrastructure: + +```sh +kubectl delete -f inflate-100-replicas-deployment.yaml --ignore-not-found +kubectl delete -f pause-50-replicas-deployment.yaml --ignore-not-found +``` + +Destroy the Terraform-managed resources: + +```sh +terraform destroy +``` + +If you built additional custom worker images with the standalone Packer flow, delete those images from OCI after they are no longer needed. Terraform only destroys images it created through the Terraform image-builder workflow. + +## Operational Guidance + +Treat optimized worker images as versioned artifacts. + +Rebuild when: + +- Oracle publishes a newer OKE worker image for the target Kubernetes version. +- Kubernetes version changes. +- CRI-O, VCN CNI, kube-proxy, or other pre-pulled image digests change. +- Security cadence requires package refresh. +- Application teams add or change application images in the pre-pull list. + +Avoid mutable tags such as `latest` in pre-pulled images. Prefer region-correct digest-pinned references. + +## Important Caveats + +- Do not run runtime `systemctl start crio` before OKE bootstrap. Testing showed that runtime CRI-O manipulation can race with OKE service ordering. +- Do not rely on a single optimized image indefinitely. Baked packages and CRI-O image layers become stale over time. +- Do not publish Terraform state, plans, private variables, or SSH keys. +- Restrict `allowed_kubernetes_api_cidrs` and `image_builder_ssh_cidr` before using outside a short-lived test environment. +- Validate every promoted image with node join, CNI, kube-proxy, DaemonSet readiness, and workload scheduling tests. + +## Documentation + +See [OKE_OPTIMIZED_WORKER_IMAGE_GUIDE.md](./OKE_OPTIMIZED_WORKER_IMAGE_GUIDE.md) for the full implementation guide, rationale, and boot-speed findings. diff --git a/oke-karpenter-fast-start-optimizer-sample/data.tf b/oke-karpenter-fast-start-optimizer-sample/data.tf new file mode 100644 index 0000000..0bf6746 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/data.tf @@ -0,0 +1,24 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +data "oci_identity_availability_domains" "ads" { + compartment_id = var.tenancy_ocid +} + +data "oci_core_images" "oke_worker" { + count = var.oke_worker_image_id == null ? 1 : 0 + + compartment_id = var.oke_image_compartment_ocid + display_name = var.oke_worker_image_display_name + shape = var.bootstrap_node_shape + sort_by = "TIMECREATED" + sort_order = "DESC" + state = "AVAILABLE" +} + +data "oci_containerengine_cluster_kube_config" "this" { + cluster_id = oci_containerengine_cluster.this.id + endpoint = "PUBLIC_ENDPOINT" + token_version = "2.0.0" + expiration = 2592000 +} diff --git a/oke-karpenter-fast-start-optimizer-sample/iam.tf b/oke-karpenter-fast-start-optimizer-sample/iam.tf new file mode 100644 index 0000000..feeaf46 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/iam.tf @@ -0,0 +1,43 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +resource "oci_identity_dynamic_group" "karpenter_nodes" { + compartment_id = var.tenancy_ocid + description = "Self-managed OKE worker nodes launched by Karpenter in ${var.project_compartment_name}." + matching_rule = "ALL {instance.compartment.id = '${var.project_compartment_ocid}'}" + name = "${var.name}-karpenter-nodes" +} + +resource "oci_identity_policy" "karpenter_workload" { + compartment_id = var.parent_compartment_ocid + description = "Workload identity permissions for Karpenter Provider OCI." + name = "${var.name}-karpenter-workload" + + statements = [ + "Allow any-user to manage instance-family in compartment ${var.project_compartment_name} where all {request.principal.type = 'workload', request.principal.namespace = '${var.karpenter_namespace}', request.principal.service_account = '${var.karpenter_service_account_name}', request.principal.cluster_id = '${oci_containerengine_cluster.this.id}'}", + "Allow any-user to manage volumes in compartment ${var.project_compartment_name} where all {request.principal.type = 'workload', request.principal.namespace = '${var.karpenter_namespace}', request.principal.service_account = '${var.karpenter_service_account_name}', request.principal.cluster_id = '${oci_containerengine_cluster.this.id}'}", + "Allow any-user to manage volume-attachments in compartment ${var.project_compartment_name} where all {request.principal.type = 'workload', request.principal.namespace = '${var.karpenter_namespace}', request.principal.service_account = '${var.karpenter_service_account_name}', request.principal.cluster_id = '${oci_containerengine_cluster.this.id}'}", + "Allow any-user to manage virtual-network-family in compartment ${var.project_compartment_name} where all {request.principal.type = 'workload', request.principal.namespace = '${var.karpenter_namespace}', request.principal.service_account = '${var.karpenter_service_account_name}', request.principal.cluster_id = '${oci_containerengine_cluster.this.id}'}", + "Allow any-user to read instance-images in compartment ${var.project_compartment_name} where all {request.principal.type = 'workload', request.principal.namespace = '${var.karpenter_namespace}', request.principal.service_account = '${var.karpenter_service_account_name}', request.principal.cluster_id = '${oci_containerengine_cluster.this.id}'}" + ] +} + +resource "oci_identity_policy" "karpenter_nodes_join" { + compartment_id = var.parent_compartment_ocid + description = "Allow Karpenter-launched self-managed nodes to join the OKE cluster." + name = "${var.name}-karpenter-node-join" + + statements = [ + "Allow dynamic-group id ${oci_identity_dynamic_group.karpenter_nodes.id} to {CLUSTER_JOIN} in compartment ${var.project_compartment_name} where target.cluster.id = '${oci_containerengine_cluster.this.id}'" + ] +} + +resource "oci_identity_policy" "karpenter_tenancy_inspect" { + compartment_id = var.tenancy_ocid + description = "Allow Karpenter workload identity to inspect compartments for OCI resource resolution." + name = "${var.name}-karpenter-tenancy-inspect" + + statements = [ + "Allow any-user to inspect compartments in tenancy where all {request.principal.type = 'workload', request.principal.namespace = '${var.karpenter_namespace}', request.principal.service_account = '${var.karpenter_service_account_name}', request.principal.cluster_id = '${oci_containerengine_cluster.this.id}'}" + ] +} diff --git a/oke-karpenter-fast-start-optimizer-sample/image_builder.tf b/oke-karpenter-fast-start-optimizer-sample/image_builder.tf new file mode 100644 index 0000000..1c4e3f9 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/image_builder.tf @@ -0,0 +1,126 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +resource "terraform_data" "image_builder_recipe" { + triggers_replace = { + base_image_id = local.worker_image_id + build_script_sha = filesha256("${path.module}/templates/image-builder-cloud-init.sh.tftpl") + prepull_images_hash = sha256(jsonencode(var.prepull_images)) + } +} + +resource "oci_core_instance" "image_builder" { + availability_domain = data.oci_identity_availability_domains.ads.availability_domains[0].name + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-image-builder" + freeform_tags = merge(local.common_tags, { Role = "image-builder" }) + shape = var.image_builder_shape + + create_vnic_details { + assign_public_ip = true + display_name = "${var.name}-image-builder" + hostname_label = "imagebuilder" + subnet_id = oci_core_subnet.image_builder.id + } + + metadata = { + ssh_authorized_keys = local.ssh_public_key + user_data = base64encode(templatefile("${path.module}/templates/image-builder-cloud-init.sh.tftpl", { + prepull_images = var.prepull_images + })) + } + + shape_config { + memory_in_gbs = var.image_builder_memory_in_gbs + ocpus = var.image_builder_ocpus + } + + source_details { + boot_volume_size_in_gbs = var.worker_boot_volume_size_in_gbs + source_id = local.worker_image_id + source_type = "image" + } + + preserve_boot_volume = false + + lifecycle { + replace_triggered_by = [terraform_data.image_builder_recipe] + } +} + +resource "terraform_data" "image_builder_ready" { + triggers_replace = { + instance_id = oci_core_instance.image_builder.id + recipe_id = terraform_data.image_builder_recipe.id + } + + connection { + host = oci_core_instance.image_builder.public_ip + private_key = file(pathexpand(var.ssh_private_key_path)) + timeout = "45m" + type = "ssh" + user = "opc" + } + + provisioner "remote-exec" { + inline = [ + "set -e", + "for i in $(seq 1 180); do if sudo test -f /var/lib/oke-optimization/image-build-complete; then sudo tail -n 120 /var/log/oke-optimized-image-build.log || true; exit 0; fi; sleep 10; done", + "sudo tail -n 200 /var/log/oke-optimized-image-build.log || true", + "echo 'Timed out waiting for optimized OKE image build marker.'", + "exit 1" + ] + } +} + +resource "terraform_data" "image_builder_stopped" { + triggers_replace = { + instance_id = oci_core_instance.image_builder.id + ready_id = terraform_data.image_builder_ready.id + } + + provisioner "local-exec" { + interpreter = ["/bin/bash", "-c"] + command = <<-EOT + set -euo pipefail + + oci compute instance action \ + --instance-id '${oci_core_instance.image_builder.id}' \ + --action SOFTSTOP \ + --profile '${var.config_file_profile}' \ + --region '${var.region}' >/dev/null + + for i in $(seq 1 120); do + state=$(oci compute instance get \ + --instance-id '${oci_core_instance.image_builder.id}' \ + --profile '${var.config_file_profile}' \ + --region '${var.region}' \ + --query 'data."lifecycle-state"' \ + --raw-output) + + if [ "$state" = "STOPPED" ]; then + exit 0 + fi + + sleep 10 + done + + echo "Timed out waiting for image builder instance to stop." + exit 1 + EOT + } +} + +resource "oci_core_image" "optimized_oke_worker" { + compartment_id = var.project_compartment_ocid + display_name = local.optimized_worker_image_display_name + freeform_tags = local.optimized_worker_image_tags + instance_id = oci_core_instance.image_builder.id + + depends_on = [terraform_data.image_builder_stopped] + + timeouts { + create = "2h" + delete = "2h" + } +} diff --git a/oke-karpenter-fast-start-optimizer-sample/inflate-100-replicas-deployment.yaml b/oke-karpenter-fast-start-optimizer-sample/inflate-100-replicas-deployment.yaml new file mode 100644 index 0000000..15a7d21 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/inflate-100-replicas-deployment.yaml @@ -0,0 +1,34 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: inflate-100-replicas + labels: + app: inflate-100-replicas +spec: + replicas: 100 + selector: + matchLabels: + app: inflate-100-replicas + template: + metadata: + labels: + app: inflate-100-replicas + spec: + nodeSelector: + oke-optimization.oracle.com/profile: fast-start + tolerations: + - key: oke-optimization.oracle.com/scaling-test + operator: Equal + value: "true" + effect: NoSchedule + terminationGracePeriodSeconds: 0 + containers: + - name: pause + image: registry.k8s.io/pause:3.9 + resources: + requests: + cpu: 500m + memory: 512Mi diff --git a/oke-karpenter-fast-start-optimizer-sample/karpenter.tf b/oke-karpenter-fast-start-optimizer-sample/karpenter.tf new file mode 100644 index 0000000..964c973 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/karpenter.tf @@ -0,0 +1,149 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +resource "helm_release" "karpenter" { + name = "karpenter" + repository = "https://oracle.github.io/karpenter-provider-oci/charts" + chart = "karpenter" + version = var.karpenter_chart_version + namespace = var.karpenter_namespace + create_namespace = true + wait = true + timeout = 900 + + values = [ + yamlencode({ + replicaCount = var.karpenter_controller_replicas + + serviceAccount = { + create = true + name = var.karpenter_service_account_name + } + + controller = { + env = [ + { + name = "OCI_RESOURCE_PRINCIPAL_VERSION" + value = "2.2" + }, + { + name = "OCI_REGION" + value = var.region + } + ] + } + + settings = { + clusterCompartmentId = var.project_compartment_ocid + vcnCompartmentId = var.project_compartment_ocid + preBakedImageCompartmentId = var.oke_image_compartment_ocid + apiserverEndpoint = local.cluster_private_endpoint + ociVcnIpNative = var.cni_type == "OCI_VCN_IP_NATIVE" + } + }) + ] + + depends_on = [ + oci_containerengine_node_pool.bootstrap, + oci_identity_policy.karpenter_nodes_join, + oci_identity_policy.karpenter_tenancy_inspect, + oci_identity_policy.karpenter_workload + ] +} + +resource "kubectl_manifest" "optimized_oci_node_class" { + yaml_body = yamlencode({ + apiVersion = "oci.oraclecloud.com/v1beta1" + kind = "OCINodeClass" + metadata = { + name = var.karpenter_nodepool_name + } + spec = { + nodeCompartmentId = var.project_compartment_ocid + sshAuthorizedKeys = [local.ssh_public_key] + shapeConfigs = local.karpenter_shape_configs + + metadata = { + user_data = local.optimized_node_user_data + } + + volumeConfig = { + bootVolumeConfig = { + sizeInGbs = var.worker_boot_volume_size_in_gbs + imageConfig = { + imageType = "OKEImage" + imageId = oci_core_image.optimized_oke_worker.id + } + } + } + + networkConfig = local.ocinodeclass_network_config + + freeformTags = local.common_tags + } + }) + + depends_on = [helm_release.karpenter] +} + +resource "kubectl_manifest" "optimized_node_pool" { + yaml_body = yamlencode({ + apiVersion = "karpenter.sh/v1" + kind = "NodePool" + metadata = { + name = var.karpenter_nodepool_name + } + spec = { + template = { + metadata = { + labels = { + "oke-optimization.oracle.com/profile" = "fast-start" + } + } + spec = { + expireAfter = "Never" + nodeClassRef = { + group = "oci.oraclecloud.com" + kind = "OCINodeClass" + name = var.karpenter_nodepool_name + } + requirements = [ + { + key = "karpenter.sh/capacity-type" + operator = "In" + values = ["on-demand"] + }, + { + key = "oci.oraclecloud.com/instance-shape" + operator = "In" + values = var.karpenter_worker_shapes + } + ] + taints = [ + { + key = "oke-optimization.oracle.com/scaling-test" + value = "true" + effect = "NoSchedule" + } + ] + terminationGracePeriod = "2m" + } + } + disruption = { + budgets = [ + { + nodes = "100%" + } + ] + consolidateAfter = "30s" + consolidationPolicy = "WhenEmptyOrUnderutilized" + } + limits = { + cpu = var.karpenter_node_cpu_limit + memory = var.karpenter_node_memory_limit + } + } + }) + + depends_on = [kubectl_manifest.optimized_oci_node_class] +} diff --git a/oke-karpenter-fast-start-optimizer-sample/locals.tf b/oke-karpenter-fast-start-optimizer-sample/locals.tf new file mode 100644 index 0000000..eb88190 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/locals.tf @@ -0,0 +1,83 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +locals { + ssh_public_key = file(pathexpand(var.ssh_public_key_path)) + + worker_image_id = var.oke_worker_image_id != null ? var.oke_worker_image_id : data.oci_core_images.oke_worker[0].images[0].id + + kubeconfig = yamldecode(data.oci_containerengine_cluster_kube_config.this.content) + kube_host = local.kubeconfig.clusters[0].cluster.server + kube_ca_certificate = base64decode(local.kubeconfig.clusters[0].cluster["certificate-authority-data"]) + kube_exec_args = [ + "ce", + "cluster", + "generate-token", + "--cluster-id", + oci_containerengine_cluster.this.id, + "--region", + var.region, + "--profile", + var.config_file_profile + ] + + cluster_private_endpoint = replace( + replace( + replace(oci_containerengine_cluster.this.endpoints[0].private_endpoint, "https://", ""), + "http://", + "" + ), + ":6443", + "" + ) + + karpenter_shape_configs = [ + for shape_config in var.karpenter_shape_configs : merge( + { + ocpus = shape_config.ocpus + memoryInGbs = shape_config.memory_in_gbs + }, + shape_config.baseline_ocpu_utilization == null ? {} : { + baselineOcpuUtilization = shape_config.baseline_ocpu_utilization + } + ) + ] + + ocinodeclass_network_config = merge( + { + primaryVnicConfig = { + subnetConfig = { + subnetId = oci_core_subnet.workers.id + } + } + }, + var.cni_type == "OCI_VCN_IP_NATIVE" ? { + secondaryVnicConfigs = [ + { + subnetConfig = { + subnetId = oci_core_subnet.pods.id + } + ipCount = var.karpenter_pod_vnic_ip_count + } + ] + } : {} + ) + + optimized_node_user_data = base64encode(templatefile("${path.module}/templates/karpenter-optimized-user-data.sh.tftpl", { + cluster_dns_ip = var.cluster_dns_ip + prepull_images = var.prepull_images + })) + + base_worker_image_display_name = var.oke_worker_image_id == null ? data.oci_core_images.oke_worker[0].images[0].display_name : var.oke_worker_image_display_name + optimized_worker_image_display_name = "${var.name}-ol8-oke-${replace(trimprefix(var.kubernetes_version, "v"), ".", "-")}-fast-start" + optimized_worker_image_tags = merge(local.common_tags, { + k8s_version = var.kubernetes_version + base_image_ocid = local.worker_image_id + base_image_name = local.base_worker_image_display_name + optimization_profile = "fast-start" + }) + + common_tags = merge(var.freeform_tags, { + Environment = var.name + }) +} diff --git a/oke-karpenter-fast-start-optimizer-sample/network.tf b/oke-karpenter-fast-start-optimizer-sample/network.tf new file mode 100644 index 0000000..f12f2f8 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/network.tf @@ -0,0 +1,201 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +resource "oci_core_vcn" "this" { + compartment_id = var.project_compartment_ocid + cidr_blocks = [var.vcn_cidr] + display_name = "${var.name}-vcn" + dns_label = "okeopt" + freeform_tags = local.common_tags +} + +resource "oci_core_internet_gateway" "this" { + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-igw" + enabled = true + freeform_tags = local.common_tags + vcn_id = oci_core_vcn.this.id +} + +resource "oci_core_nat_gateway" "this" { + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-nat" + freeform_tags = local.common_tags + vcn_id = oci_core_vcn.this.id +} + +resource "oci_core_service_gateway" "this" { + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-sgw" + freeform_tags = local.common_tags + services { + service_id = data.oci_core_services.all.services[0].id + } + vcn_id = oci_core_vcn.this.id +} + +data "oci_core_services" "all" { + filter { + name = "name" + values = ["All .* Services In Oracle Services Network"] + regex = true + } +} + +resource "oci_core_route_table" "public" { + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-public-rt" + freeform_tags = local.common_tags + vcn_id = oci_core_vcn.this.id + + route_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_internet_gateway.this.id + } +} + +resource "oci_core_route_table" "private" { + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-private-rt" + freeform_tags = local.common_tags + vcn_id = oci_core_vcn.this.id + + route_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + network_entity_id = oci_core_nat_gateway.this.id + } + + route_rules { + destination = data.oci_core_services.all.services[0].cidr_block + destination_type = "SERVICE_CIDR_BLOCK" + network_entity_id = oci_core_service_gateway.this.id + } +} + +resource "oci_core_security_list" "oke" { + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-security-list" + freeform_tags = local.common_tags + vcn_id = oci_core_vcn.this.id + + egress_security_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + protocol = "all" + } + + egress_security_rules { + destination = data.oci_core_services.all.services[0].cidr_block + destination_type = "SERVICE_CIDR_BLOCK" + protocol = "all" + } + + ingress_security_rules { + description = "Allow all VCN-internal OKE control plane, node, and pod traffic." + protocol = "all" + source = var.vcn_cidr + source_type = "CIDR_BLOCK" + } + + dynamic "ingress_security_rules" { + for_each = toset(var.allowed_kubernetes_api_cidrs) + content { + description = "Kubernetes API public endpoint access." + protocol = "6" + source = ingress_security_rules.value + source_type = "CIDR_BLOCK" + + tcp_options { + min = 6443 + max = 6443 + } + } + } +} + +resource "oci_core_security_list" "image_builder" { + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-image-builder-security-list" + freeform_tags = local.common_tags + vcn_id = oci_core_vcn.this.id + + egress_security_rules { + destination = "0.0.0.0/0" + destination_type = "CIDR_BLOCK" + protocol = "all" + } + + ingress_security_rules { + description = "SSH access for Terraform image-builder verification." + protocol = "6" + source = var.image_builder_ssh_cidr + source_type = "CIDR_BLOCK" + + tcp_options { + min = 22 + max = 22 + } + } +} + +resource "oci_core_subnet" "api_endpoint" { + cidr_block = var.api_endpoint_subnet_cidr + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-api-endpoint" + dns_label = "api" + freeform_tags = local.common_tags + prohibit_public_ip_on_vnic = false + route_table_id = oci_core_route_table.public.id + security_list_ids = [oci_core_security_list.oke.id] + vcn_id = oci_core_vcn.this.id +} + +resource "oci_core_subnet" "workers" { + cidr_block = var.worker_subnet_cidr + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-workers" + dns_label = "workers" + freeform_tags = local.common_tags + prohibit_public_ip_on_vnic = true + route_table_id = oci_core_route_table.private.id + security_list_ids = [oci_core_security_list.oke.id] + vcn_id = oci_core_vcn.this.id +} + +resource "oci_core_subnet" "pods" { + cidr_block = var.pod_subnet_cidr + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-pods" + dns_label = "pods" + freeform_tags = local.common_tags + prohibit_public_ip_on_vnic = true + route_table_id = oci_core_route_table.private.id + security_list_ids = [oci_core_security_list.oke.id] + vcn_id = oci_core_vcn.this.id +} + +resource "oci_core_subnet" "load_balancers" { + cidr_block = var.load_balancer_subnet_cidr + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-load-balancers" + dns_label = "lb" + freeform_tags = local.common_tags + prohibit_public_ip_on_vnic = false + route_table_id = oci_core_route_table.public.id + security_list_ids = [oci_core_security_list.oke.id] + vcn_id = oci_core_vcn.this.id +} + +resource "oci_core_subnet" "image_builder" { + cidr_block = var.image_builder_subnet_cidr + compartment_id = var.project_compartment_ocid + display_name = "${var.name}-image-builder" + dns_label = "builder" + freeform_tags = local.common_tags + prohibit_public_ip_on_vnic = false + route_table_id = oci_core_route_table.public.id + security_list_ids = [oci_core_security_list.image_builder.id] + vcn_id = oci_core_vcn.this.id +} diff --git a/oke-karpenter-fast-start-optimizer-sample/oke.tf b/oke-karpenter-fast-start-optimizer-sample/oke.tf new file mode 100644 index 0000000..c8b4f6b --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/oke.tf @@ -0,0 +1,87 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +resource "oci_containerengine_cluster" "this" { + compartment_id = var.project_compartment_ocid + kubernetes_version = var.kubernetes_version + name = var.name + type = var.cluster_type + vcn_id = oci_core_vcn.this.id + freeform_tags = local.common_tags + + cluster_pod_network_options { + cni_type = var.cni_type + } + + endpoint_config { + is_public_ip_enabled = true + subnet_id = oci_core_subnet.api_endpoint.id + } + + options { + add_ons { + is_kubernetes_dashboard_enabled = false + is_tiller_enabled = false + } + + kubernetes_network_config { + pods_cidr = var.cni_type == "FLANNEL_OVERLAY" ? "10.244.0.0/16" : null + services_cidr = "10.96.0.0/16" + } + + service_lb_subnet_ids = [oci_core_subnet.load_balancers.id] + } +} + +resource "oci_containerengine_node_pool" "bootstrap" { + cluster_id = oci_containerengine_cluster.this.id + compartment_id = var.project_compartment_ocid + freeform_tags = local.common_tags + kubernetes_version = var.kubernetes_version + name = "${var.name}-bootstrap" + node_shape = var.bootstrap_node_shape + ssh_public_key = local.ssh_public_key + + initial_node_labels { + key = "oke-optimization.oracle.com/purpose" + value = "karpenter-controller" + } + + node_config_details { + freeform_tags = local.common_tags + nsg_ids = [] + size = var.bootstrap_node_pool_size + + dynamic "placement_configs" { + for_each = data.oci_identity_availability_domains.ads.availability_domains + content { + availability_domain = placement_configs.value.name + subnet_id = oci_core_subnet.workers.id + } + } + + dynamic "node_pool_pod_network_option_details" { + for_each = var.cni_type == "OCI_VCN_IP_NATIVE" ? [1] : [] + content { + cni_type = var.cni_type + max_pods_per_node = 31 + pod_subnet_ids = [oci_core_subnet.pods.id] + } + } + } + + node_metadata = { + ssh_authorized_keys = local.ssh_public_key + } + + node_shape_config { + memory_in_gbs = var.bootstrap_node_memory_in_gbs + ocpus = var.bootstrap_node_ocpus + } + + node_source_details { + boot_volume_size_in_gbs = var.worker_boot_volume_size_in_gbs + image_id = local.worker_image_id + source_type = "IMAGE" + } +} diff --git a/oke-karpenter-fast-start-optimizer-sample/outputs.tf b/oke-karpenter-fast-start-optimizer-sample/outputs.tf new file mode 100644 index 0000000..0516abe --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/outputs.tf @@ -0,0 +1,57 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +output "project_compartment_id" { + description = "OCID of the OKEOptimization project compartment." + value = var.project_compartment_ocid +} + +output "cluster_id" { + description = "OKE cluster OCID." + value = oci_containerengine_cluster.this.id +} + +output "cluster_private_endpoint" { + description = "Private Kubernetes API endpoint passed to KPO for worker bootstrap." + value = local.cluster_private_endpoint +} + +output "cluster_public_endpoint" { + description = "Public Kubernetes API endpoint used by Terraform providers." + value = oci_containerengine_cluster.this.endpoints[0].public_endpoint +} + +output "worker_image_id" { + description = "Oracle Linux OKE worker image used by the bootstrap pool and as the custom image base." + value = local.worker_image_id +} + +output "optimized_worker_image_id" { + description = "Custom optimized Oracle Linux OKE worker image used by Karpenter." + value = oci_core_image.optimized_oke_worker.id +} + +output "optimized_worker_image_display_name" { + description = "Display name for the custom optimized Oracle Linux OKE worker image." + value = oci_core_image.optimized_oke_worker.display_name +} + +output "image_builder_instance_id" { + description = "Temporary builder instance used to create the optimized worker image." + value = oci_core_instance.image_builder.id +} + +output "worker_image_display_name" { + description = "Display name looked up for the worker image when oke_worker_image_id is null." + value = var.oke_worker_image_id == null ? data.oci_core_images.oke_worker[0].images[0].display_name : null +} + +output "karpenter_namespace" { + description = "Namespace where KPO is installed." + value = var.karpenter_namespace +} + +output "karpenter_nodepool_name" { + description = "Karpenter NodePool and OCINodeClass name." + value = var.karpenter_nodepool_name +} diff --git a/oke-karpenter-fast-start-optimizer-sample/packer/.gitignore b/oke-karpenter-fast-start-optimizer-sample/packer/.gitignore new file mode 100644 index 0000000..ff80d74 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/packer/.gitignore @@ -0,0 +1,7 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +local*.pkrvars.hcl +*.pkrvars.hcl +!*example.pkrvars.hcl +packer_cache/ diff --git a/oke-karpenter-fast-start-optimizer-sample/packer/README.md b/oke-karpenter-fast-start-optimizer-sample/packer/README.md new file mode 100644 index 0000000..4fde21f --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/packer/README.md @@ -0,0 +1,145 @@ + + + +# Packer Pipeline for Optimized OKE Worker Images + +This directory contains a Packer pipeline that takes an Oracle-published OKE worker image OCID and creates a custom optimized OKE worker image with the image-based fast-start optimizations used by the Terraform implementation. + +The Packer Oracle OCI builder launches a temporary OCI instance from `base_image_ocid`, runs the provisioner script, snapshots the result as a custom image, and terminates the temporary instance. The builder behavior is based on the official HashiCorp Oracle OCI Packer builder. + +## What This Pipeline Bakes Into the Image + +The provisioner script implements only image-time optimizations: + +- Installs `oraclelinux-developer-release-el8`. +- Installs `jq`, `curl`, and `python36-oci-cli`. +- Verifies `crictl` exists on the OKE worker image. +- Starts CRI-O only during the image build. +- Pre-pulls digest-pinned OKE infrastructure images into CRI-O storage. +- Verifies the pre-pulled image layers are present. +- Writes `/var/lib/oke-optimization/image-build-manifest.json`. +- Cleans DNF cache, transient cloud-init state, logs, shell history, and machine identity. +- Preserves `/etc/oke` and CRI-O image storage. + +It intentionally does not implement runtime bootstrap changes. Runtime bootstrap still belongs in the KPO `OCINodeClass.spec.metadata.user_data` template. + +## Required Inputs + +At minimum you must provide: + +- `base_image_ocid`: OCID of the Oracle-published OKE worker image to optimize. +- `availability_domain`: AD for the temporary builder instance. +- `subnet_ocid`: subnet that the Packer host can SSH into. +- `compartment_ocid`: compartment for the temporary builder instance. + +The example uses placeholder OCIDs. Replace them with values from your tenancy before building. + +## Usage + +Initialize the Packer plugin: + +```sh +cd packer +packer init . +``` + +Create a local var file: + +```sh +cp oke-optimized-worker.example.pkrvars.hcl local.pkrvars.hcl +``` + +Edit `local.pkrvars.hcl` and set: + +```hcl +base_image_ocid = "REPLACE_WITH_BASE_OKE_WORKER_IMAGE_OCID" +base_image_name = "Oracle-Linux-8.10-..." +compartment_ocid = "REPLACE_WITH_COMPARTMENT_OCID" +availability_domain = "..." +subnet_ocid = "REPLACE_WITH_SUBNET_OCID" +image_name = "oke-optimized-ol8-oke-1-35-2-fast-start-packer" +ssh_public_key_path = "/absolute/path/to/id_rsa.pub" +ssh_private_key_path = "/absolute/path/to/id_rsa" +``` + +Validate: + +```sh +packer validate -var-file=local.pkrvars.hcl . +``` + +Build: + +```sh +packer build -var-file=local.pkrvars.hcl . +``` + +Or use the wrapper that takes the base OKE worker image OCID and discovers builder settings from Terraform outputs and OCI CLI: + +```sh +./scripts/build-from-image-ocid.sh +``` + +For a different tenancy or standalone Packer use, provide overrides: + +```sh +PACKER_COMPARTMENT_OCID= \ +PACKER_SUBNET_OCID= \ +PACKER_AVAILABILITY_DOMAIN='xxxx:US-ASHBURN-AD-1' \ +PACKER_SSH_PUBLIC_KEY_PATH=/absolute/path/to/id_rsa.pub \ +PACKER_SSH_PRIVATE_KEY_PATH=/absolute/path/to/id_rsa \ +./scripts/build-from-image-ocid.sh +``` + +To validate the generated Packer variables without building an image: + +```sh +PACKER_VALIDATE_ONLY=true ./scripts/build-from-image-ocid.sh +``` + +For a script-only test that provisions the builder but does not create an image, set: + +```hcl +skip_create_image = true +``` + +## Example One-Off Build + +```sh +packer build \ + -var 'base_image_ocid=' \ + -var 'base_image_name=Oracle-Linux-8.10-2026.02.28-0-OKE-1.35.2-1402' \ + -var 'availability_domain=REPLACE_WITH_AD_NAME' \ + -var 'subnet_ocid=REPLACE_WITH_IMAGE_BUILDER_SUBNET_OCID' \ + -var 'image_name=oke-optimized-ol8-oke-1-35-2-fast-start-packer' \ + . +``` + +## Drift and Rebuild Guidance + +This pipeline freezes packages and pre-pulled image layers at build time. That is what makes node boot faster, but it also means the optimized image must be rebuilt. + +Rebuild when: + +- Oracle publishes a newer OKE worker image for the target Kubernetes version. +- Any OKE infrastructure image digest changes. +- The Kubernetes target version changes. +- A security cadence requires package refresh, for example monthly. +- Application teams change any app images added to `prepull_images`. + +Avoid mutable `latest` tags in `prepull_images`. Use region-correct digest-pinned image references so the image build is deterministic. + +Prefer rebuilding from Oracle's latest OKE worker image over running broad, untested `dnf update -y` on an older worker image. OKE worker images are curated artifacts; package-level mutation should be validated with a node-join and workload scheduling test before promotion. + +## Promotion Checklist + +After Packer creates the image: + +1. Confirm the image is `AVAILABLE` in OCI. +2. Confirm tags include `k8s_version`, `base_image_ocid`, `base_image_name`, `optimization_profile`, and `prepull_images_sha256`. +3. Update the KPO `OCINodeClass` image ID to the new custom image OCID. +4. Launch one Karpenter node from the new image. +5. Confirm the node joins Ready. +6. Confirm `/var/lib/oke-optimization/image-build-manifest.json` exists on the node. +7. Run a small workload pinned to `oke-optimization.oracle.com/profile=fast-start`. +8. Run the larger scale test after the smoke test passes. diff --git a/oke-karpenter-fast-start-optimizer-sample/packer/oke-optimized-worker.example.pkrvars.hcl b/oke-karpenter-fast-start-optimizer-sample/packer/oke-optimized-worker.example.pkrvars.hcl new file mode 100644 index 0000000..ad4cd7c --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/packer/oke-optimized-worker.example.pkrvars.hcl @@ -0,0 +1,37 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +# Copy this file to a local .pkrvars.hcl file and fill in the environment-specific values. +# Do not commit private keys or other secrets. + +base_image_ocid = "REPLACE_WITH_OKE_WORKER_IMAGE_OCID" +base_image_name = "Oracle-Linux-8.10-YYYY.MM.DD-0-OKE-1.35.x-xxxx" + +region = "us-ashburn-1" +oci_profile = "DEFAULT" +compartment_ocid = "REPLACE_WITH_COMPARTMENT_OCID" +availability_domain = "REPLACE_WITH_AD_NAME" +subnet_ocid = "REPLACE_WITH_IMAGE_BUILDER_SUBNET_OCID" + +image_name = "oke-optimized-ol8-oke-1-35-2-fast-start-packer" +kubernetes_version = "v1.35.2" + +shape = "VM.Standard.E5.Flex" +ocpus = 4 +memory_in_gbs = 16 +boot_volume_size_in_gbs = 50 +assign_public_ip = true +ssh_public_key_path = "/absolute/path/to/id_rsa.pub" +ssh_private_key_path = "/absolute/path/to/id_rsa" +optimization_profile = "fast-start" +skip_create_image = false + +prepull_images = [ + "iad.ocir.io/axoxdievda5j/oke-public-pause@sha256:6e287874898efea60adff9dc1faaee355740b192763f963fe66cf755d995cf32", + "iad.ocir.io/id9y6mi8tcky/oke-public-vcn-native-ip-cni-plugin@sha256:916f9953b84f2788b410c45ba766901e99d9c345d37e2fc35a16c312f27e34f0", + "iad.ocir.io/id9y6mi8tcky/oke-public-kube-proxy@sha256:175559244baa3cbee68adfc1357c18871406c7a2d49dd487d2f04e111e7931a0" +] + +extra_image_tags = { + BuildTool = "packer" +} diff --git a/oke-karpenter-fast-start-optimizer-sample/packer/oke-optimized-worker.pkr.hcl b/oke-karpenter-fast-start-optimizer-sample/packer/oke-optimized-worker.pkr.hcl new file mode 100644 index 0000000..7dfc195 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/packer/oke-optimized-worker.pkr.hcl @@ -0,0 +1,222 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +packer { + required_plugins { + oracle = { + source = "github.com/hashicorp/oracle" + version = ">= 1.1.2" + } + } +} + +variable "base_image_ocid" { + type = string + description = "OCID of the Oracle-published OKE worker image to optimize." +} + +variable "base_image_name" { + type = string + description = "Display name of the base OKE worker image. Used only for image tags and the build manifest." + default = "unknown" +} + +variable "image_name" { + type = string + description = "Display name for the resulting optimized custom image. If empty, a timestamped name is generated." + default = "" +} + +variable "compartment_ocid" { + type = string + description = "Compartment where the temporary Packer builder instance runs." +} + +variable "image_compartment_ocid" { + type = string + description = "Compartment where the optimized image is created. Defaults to compartment_ocid when empty." + default = "" +} + +variable "availability_domain" { + type = string + description = "Availability domain where the temporary builder instance runs, for example Uocm:US-ASHBURN-AD-1." +} + +variable "subnet_ocid" { + type = string + description = "Subnet OCID for the temporary builder instance. It must allow SSH from the Packer host." +} + +variable "region" { + type = string + description = "OCI region." + default = "us-ashburn-1" +} + +variable "oci_profile" { + type = string + description = "OCI config profile used by the Packer Oracle builder." + default = "DEFAULT" +} + +variable "shape" { + type = string + description = "Builder instance shape." + default = "VM.Standard.E5.Flex" +} + +variable "ocpus" { + type = number + description = "Builder instance OCPUs for flexible shapes." + default = 4 +} + +variable "memory_in_gbs" { + type = number + description = "Builder instance memory for flexible shapes." + default = 16 +} + +variable "boot_volume_size_in_gbs" { + type = number + description = "Builder boot volume size. OCI requires at least 50 GB." + default = 50 +} + +variable "assign_public_ip" { + type = bool + description = "Assign a public IP to the temporary builder VNIC. Set false only if Packer can SSH over private networking." + default = true +} + +variable "ssh_username" { + type = string + description = "SSH user for the OKE worker image." + default = "opc" +} + +variable "ssh_public_key_path" { + type = string + description = "SSH public key injected into the temporary builder instance." +} + +variable "ssh_private_key_path" { + type = string + description = "SSH private key used by Packer to connect to the temporary builder instance." +} + +variable "kubernetes_version" { + type = string + description = "Target OKE Kubernetes version for image tagging." + default = "v1.35.2" +} + +variable "optimization_profile" { + type = string + description = "Optimization profile tag value." + default = "fast-start" +} + +variable "project_name" { + type = string + description = "Project tag value." + default = "OKEOptimization" +} + +variable "skip_create_image" { + type = bool + description = "Set true for build-script testing without creating a custom image." + default = false +} + +variable "prepull_images" { + type = list(string) + description = "OCI/OKE images to pre-pull into CRI-O storage during image build. Use digest-pinned, region-correct references." + default = [ + "iad.ocir.io/axoxdievda5j/oke-public-pause@sha256:6e287874898efea60adff9dc1faaee355740b192763f963fe66cf755d995cf32", + "iad.ocir.io/id9y6mi8tcky/oke-public-vcn-native-ip-cni-plugin@sha256:916f9953b84f2788b410c45ba766901e99d9c345d37e2fc35a16c312f27e34f0", + "iad.ocir.io/id9y6mi8tcky/oke-public-kube-proxy@sha256:175559244baa3cbee68adfc1357c18871406c7a2d49dd487d2f04e111e7931a0" + ] +} + +variable "extra_image_tags" { + type = map(string) + description = "Additional freeform tags to place on the resulting optimized image." + default = {} +} + +locals { + generated_image_name = "oke-optimized-ol8-${replace(trimprefix(var.kubernetes_version, "v"), ".", "-")}-${var.optimization_profile}-${formatdate("YYYYMMDD-hhmmss", timestamp())}" + image_name = var.image_name != "" ? var.image_name : local.generated_image_name + image_compartment_ocid = var.image_compartment_ocid != "" ? var.image_compartment_ocid : var.compartment_ocid + prepull_images_fingerprint = sha256(jsonencode(var.prepull_images)) + + common_tags = { + ManagedBy = "packer" + Project = var.project_name + k8s_version = var.kubernetes_version + base_image_ocid = var.base_image_ocid + base_image_name = var.base_image_name + optimization_profile = var.optimization_profile + prepull_images_sha256 = local.prepull_images_fingerprint + } +} + +source "oracle-oci" "oke_optimized_worker" { + access_cfg_file_account = var.oci_profile + region = var.region + + availability_domain = var.availability_domain + base_image_ocid = var.base_image_ocid + compartment_ocid = var.compartment_ocid + image_compartment_ocid = local.image_compartment_ocid + image_name = local.image_name + instance_name = "${local.image_name}-builder" + + shape = var.shape + shape_config { + ocpus = var.ocpus + memory_in_gbs = var.memory_in_gbs + } + + disk_size = var.boot_volume_size_in_gbs + + ssh_username = var.ssh_username + ssh_private_key_file = var.ssh_private_key_path + subnet_ocid = var.subnet_ocid + + create_vnic_details { + assign_public_ip = var.assign_public_ip + } + + metadata = { + ssh_authorized_keys = file(var.ssh_public_key_path) + } + + instance_options_are_legacy_imds_endpoints_disabled = true + skip_create_image = var.skip_create_image + + instance_tags = merge(local.common_tags, { + Role = "packer-image-builder" + }) + + tags = merge(local.common_tags, var.extra_image_tags) +} + +build { + name = "oke-optimized-worker" + sources = ["source.oracle-oci.oke_optimized_worker"] + + provisioner "shell" { + environment_vars = [ + "BASE_IMAGE_OCID=${var.base_image_ocid}", + "BASE_IMAGE_NAME=${var.base_image_name}", + "KUBERNETES_VERSION=${var.kubernetes_version}", + "OPTIMIZATION_PROFILE=${var.optimization_profile}", + "PREPULL_IMAGES=${join(" ", var.prepull_images)}" + ] + pause_before = "10s" + script = "${path.root}/scripts/optimize-oke-worker.sh" + } +} diff --git a/oke-karpenter-fast-start-optimizer-sample/packer/scripts/build-from-image-ocid.sh b/oke-karpenter-fast-start-optimizer-sample/packer/scripts/build-from-image-ocid.sh new file mode 100644 index 0000000..6e3a089 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/packer/scripts/build-from-image-ocid.sh @@ -0,0 +1,125 @@ +#!/usr/bin/env bash + +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +set -euo pipefail + +PACKER_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PROJECT_DIR="$(cd "${PACKER_DIR}/.." && pwd)" + +usage() { + cat <<'USAGE' +Usage: + build-from-image-ocid.sh [image-name] + +Environment overrides: + OCI_PROFILE OCI CLI/Packer profile. Default: DEFAULT + OCI_REGION OCI region. Default: us-ashburn-1 + PACKER_TENANCY_OCID Optional tenancy OCID used to discover availability domains. + PACKER_COMPARTMENT_OCID Builder/image compartment. Defaults to terraform output project_compartment_id. + PACKER_IMAGE_COMPARTMENT_OCID Image target compartment. Defaults to PACKER_COMPARTMENT_OCID. + PACKER_SUBNET_OCID Builder subnet. Defaults to subnet named oke-optimization-image-builder. + PACKER_AVAILABILITY_DOMAIN Builder AD. Defaults to the first AD returned by OCI. + PACKER_SSH_PUBLIC_KEY_PATH SSH public key path. Default: $HOME/.ssh/id_rsa.pub + PACKER_SSH_PRIVATE_KEY_PATH SSH private key path. Default: $HOME/.ssh/id_rsa + KUBERNETES_VERSION Tag value. Default: v1.35.2 + PACKER_VALIDATE_ONLY Set to true to validate instead of build. +USAGE +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +BASE_IMAGE_OCID="${1:-}" +if [[ -z "${BASE_IMAGE_OCID}" ]]; then + usage + exit 1 +fi + +PROFILE="${OCI_PROFILE:-DEFAULT}" +REGION="${OCI_REGION:-us-ashburn-1}" +TENANCY_OCID="${PACKER_TENANCY_OCID:-}" +K8S_VERSION="${KUBERNETES_VERSION:-v1.35.2}" +K8S_VERSION_SLUG="${K8S_VERSION#v}" +K8S_VERSION_SLUG="${K8S_VERSION_SLUG//./-}" +SSH_PUBLIC_KEY_PATH="${PACKER_SSH_PUBLIC_KEY_PATH:-${HOME}/.ssh/id_rsa.pub}" +SSH_PRIVATE_KEY_PATH="${PACKER_SSH_PRIVATE_KEY_PATH:-${HOME}/.ssh/id_rsa}" + +if [[ -n "${PACKER_COMPARTMENT_OCID:-}" ]]; then + COMPARTMENT_OCID="${PACKER_COMPARTMENT_OCID}" +else + COMPARTMENT_OCID="$(terraform -chdir="${PROJECT_DIR}" output -raw project_compartment_id)" +fi + +IMAGE_COMPARTMENT_OCID="${PACKER_IMAGE_COMPARTMENT_OCID:-${COMPARTMENT_OCID}}" + +if [[ -n "${PACKER_SUBNET_OCID:-}" ]]; then + SUBNET_OCID="${PACKER_SUBNET_OCID}" +else + SUBNET_OCID="$(oci network subnet list \ + --compartment-id "${COMPARTMENT_OCID}" \ + --display-name "oke-optimization-image-builder" \ + --profile "${PROFILE}" \ + --region "${REGION}" \ + --query 'data[0].id' \ + --raw-output)" +fi + +if [[ -n "${PACKER_AVAILABILITY_DOMAIN:-}" ]]; then + AVAILABILITY_DOMAIN="${PACKER_AVAILABILITY_DOMAIN}" +else + AD_COMPARTMENT_OCID="${TENANCY_OCID:-${COMPARTMENT_OCID}}" + AVAILABILITY_DOMAIN="$(oci iam availability-domain list \ + --compartment-id "${AD_COMPARTMENT_OCID}" \ + --profile "${PROFILE}" \ + --region "${REGION}" \ + --query 'data[0].name' \ + --raw-output)" +fi + +BASE_IMAGE_NAME="$(oci compute image get \ + --image-id "${BASE_IMAGE_OCID}" \ + --profile "${PROFILE}" \ + --region "${REGION}" \ + --query 'data."display-name"' \ + --raw-output 2>/dev/null || true)" +BASE_IMAGE_NAME="${BASE_IMAGE_NAME:-unknown}" + +DEFAULT_IMAGE_NAME="oke-optimized-ol8-oke-${K8S_VERSION_SLUG}-fast-start-packer-$(date +%Y%m%d%H%M%S)" +IMAGE_NAME="${2:-${DEFAULT_IMAGE_NAME}}" + +VAR_FILE="$(mktemp "${PACKER_DIR}/local.auto.XXXXXX.pkrvars.hcl")" +trap 'rm -f "${VAR_FILE}"' EXIT + +cat > "${VAR_FILE}" < >(tee -a "${LOG_FILE}") 2>&1 + +echo "Starting optimized OKE image build at $(date -Is)" +echo "Base image OCID: ${BASE_IMAGE_OCID:-unknown}" +echo "Base image name: ${BASE_IMAGE_NAME:-unknown}" +echo "Kubernetes version: ${KUBERNETES_VERSION:-unknown}" +echo "Optimization profile: ${OPTIMIZATION_PROFILE:-fast-start}" + +echo "Installing packages that must not be downloaded during Karpenter node boot..." +dnf -y install oraclelinux-developer-release-el8 +dnf -y install jq curl python36-oci-cli + +if ! command -v crictl >/dev/null 2>&1; then + echo "crictl is required on the OKE worker image but was not found." + exit 1 +fi + +if [[ -z "${PREPULL_IMAGES:-}" ]]; then + echo "PREPULL_IMAGES must contain at least one image reference." + exit 1 +fi + +read -r -a PULL_IMAGES <<< "${PREPULL_IMAGES}" + +echo "Starting CRI-O for image pre-pulls..." +systemctl start crio + +echo "Pre-pulling OKE bootstrap images into CRI-O storage..." +pull_pids=() +for image in "${PULL_IMAGES[@]}"; do + ( + echo "Pulling ${image}" + crictl pull "${image}" + ) & + pull_pids+=("$!") +done + +for pid in "${pull_pids[@]}"; do + wait "${pid}" +done + +echo "Verifying pre-pulled images..." +for image in "${PULL_IMAGES[@]}"; do + image_repo="${image%@*}" + if [[ "${image}" == *@sha256:* ]]; then + image_digest="${image##*@}" + crictl images --digests | grep -F "${image_repo}" | grep -F "${image_digest}" >/dev/null + else + crictl images --digests | grep -F "${image_repo}" >/dev/null + fi +done + +echo "Writing image build manifest..." +printf '%s\n' "${PULL_IMAGES[@]}" | + jq -R . | + jq -s \ + --arg build_time "$(date -Is)" \ + --arg base_image_ocid "${BASE_IMAGE_OCID:-unknown}" \ + --arg base_image_name "${BASE_IMAGE_NAME:-unknown}" \ + --arg kubernetes_version "${KUBERNETES_VERSION:-unknown}" \ + --arg optimization_profile "${OPTIMIZATION_PROFILE:-fast-start}" \ + '{ + build_time: $build_time, + base_image_ocid: $base_image_ocid, + base_image_name: $base_image_name, + kubernetes_version: $kubernetes_version, + optimization_profile: $optimization_profile, + prepulled_images: . + }' > "${MANIFEST_FILE}" + +echo "Cleaning transient image-builder state while preserving OKE bootstrap assets and CRI-O image layers..." +dnf clean all +rm -rf /var/cache/dnf/* /var/tmp/* /tmp/* +rm -rf /var/lib/cloud/instances/* /var/lib/cloud/instance +rm -f /root/.bash_history /home/opc/.bash_history +find /var/log -type f ! -name "oke-optimized-image-build.log" -exec truncate -s 0 {} \; || true +truncate -s 0 /etc/machine-id || true + +sync +touch "${MARKER_FILE}" +echo "Optimized OKE image build complete at $(date -Is)" diff --git a/oke-karpenter-fast-start-optimizer-sample/pause-50-replicas-deployment.yaml b/oke-karpenter-fast-start-optimizer-sample/pause-50-replicas-deployment.yaml new file mode 100644 index 0000000..d85d9df --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/pause-50-replicas-deployment.yaml @@ -0,0 +1,33 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pause-50-replicas + labels: + app: pause-50-replicas +spec: + replicas: 50 + selector: + matchLabels: + app: pause-50-replicas + template: + metadata: + labels: + app: pause-50-replicas + spec: + nodeSelector: + oke-optimization.oracle.com/profile: fast-start + tolerations: + - key: oke-optimization.oracle.com/scaling-test + operator: Equal + value: "true" + effect: NoSchedule + containers: + - name: pause + image: registry.k8s.io/pause:3.9 + resources: + requests: + cpu: 500m + memory: 512Mi diff --git a/oke-karpenter-fast-start-optimizer-sample/templates/image-builder-cloud-init.sh.tftpl b/oke-karpenter-fast-start-optimizer-sample/templates/image-builder-cloud-init.sh.tftpl new file mode 100644 index 0000000..b63418d --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/templates/image-builder-cloud-init.sh.tftpl @@ -0,0 +1,65 @@ +#!/usr/bin/env bash + +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +set -euo pipefail + +LOG_FILE="/var/log/oke-optimized-image-build.log" +MARKER_DIR="/var/lib/oke-optimization" +MARKER_FILE="$${MARKER_DIR}/image-build-complete" + +mkdir -p "$${MARKER_DIR}" +exec > >(tee -a "$${LOG_FILE}") 2>&1 + +echo "Starting optimized OKE image build at $(date -Is)" + +echo "Installing packages that must not be downloaded during Karpenter node boot..." +dnf -y install oraclelinux-developer-release-el8 +dnf -y install jq curl python36-oci-cli + +if ! command -v crictl >/dev/null 2>&1; then + echo "crictl is required on the OKE worker image but was not found." + exit 1 +fi + +echo "Starting CRI-O for image pre-pulls..." +systemctl start crio + +PULL_IMAGES=( +%{ for image in prepull_images ~} + "${image}" +%{ endfor ~} +) + +echo "Pre-pulling OKE bootstrap images into CRI-O storage..." +pull_pids=() +for image in "$${PULL_IMAGES[@]}"; do + ( + echo "Pulling $${image}" + crictl pull "$${image}" + ) & + pull_pids+=("$!") +done + +for pid in "$${pull_pids[@]}"; do + wait "$${pid}" +done + +echo "Verifying pre-pulled images..." +for image in "$${PULL_IMAGES[@]}"; do + image_repo="$${image%@*}" + crictl images --digests | grep -F "$${image_repo}" >/dev/null +done + +echo "Cleaning transient image-builder state while preserving OKE bootstrap assets and CRI-O image layers..." +dnf clean all +rm -rf /var/cache/dnf/* /var/tmp/* /tmp/* +rm -rf /var/lib/cloud/instances/* /var/lib/cloud/instance +rm -f /root/.bash_history /home/opc/.bash_history +find /var/log -type f ! -name "oke-optimized-image-build.log" -exec truncate -s 0 {} \; || true +truncate -s 0 /etc/machine-id || true + +sync +touch "$${MARKER_FILE}" +echo "Optimized OKE image build complete at $(date -Is)" diff --git a/oke-karpenter-fast-start-optimizer-sample/templates/karpenter-optimized-user-data.sh.tftpl b/oke-karpenter-fast-start-optimizer-sample/templates/karpenter-optimized-user-data.sh.tftpl new file mode 100644 index 0000000..444fe60 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/templates/karpenter-optimized-user-data.sh.tftpl @@ -0,0 +1,102 @@ +#!/usr/bin/env bash + +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +set -euo pipefail + +TIMING_DIR="/var/lib/oke-optimization" +TIMING_LOG="/var/log/oke-optimized-bootstrap-timing.log" +mkdir -p "$TIMING_DIR" +touch "$TIMING_LOG" + +mark() { + local event="$1" + local ts epoch uptime_seconds + ts=$(date -Is) + epoch=$(date +%s) + uptime_seconds=$(cut -d' ' -f1 /proc/uptime) + printf '%s epoch=%s uptime_seconds=%s event=%s\n' "$ts" "$epoch" "$uptime_seconds" "$event" | tee -a "$TIMING_LOG" + printf '%s\n' "$ts" > "$TIMING_DIR/$event" +} + +wait_for_kubelet_active() { + local deadline + deadline=$((SECONDS + 300)) + while (( SECONDS < deadline )); do + if systemctl is-active --quiet kubelet; then + mark "kubelet-active" + return 0 + fi + sleep 2 + done + mark "kubelet-active-timeout" + return 0 +} + +wait_for_node_ready() { + local api_server ca_file cert_file deadline node_name + deadline=$((SECONDS + 300)) + ca_file="/etc/kubernetes/ca.crt" + cert_file="/var/lib/kubelet/pki/kubelet-client-current.pem" + node_name=$(hostname -I 2>/dev/null | awk '{print $1}') + if [[ -z "$node_name" ]]; then + node_name=$(hostname -f 2>/dev/null || hostname) + fi + + while (( SECONDS < deadline )); do + if [[ -s /etc/kubernetes/kubelet.conf && -s "$ca_file" && -s "$cert_file" ]]; then + api_server=$(awk '/server: / {print $2; exit}' /etc/kubernetes/kubelet.conf) + if [[ -n "$api_server" ]]; then + if curl -sfL --max-time 5 --cacert "$ca_file" --cert "$cert_file" --key "$cert_file" \ + "$api_server/api/v1/nodes/$node_name" 2>/dev/null | + jq -er '.status.conditions[] | select(.type == "Ready" and .status == "True")' >/dev/null; then + mark "node-ready" + return 0 + fi + fi + fi + + if command -v kubectl >/dev/null 2>&1; then + if kubectl get node "$node_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -q True; then + mark "node-ready" + return 0 + fi + fi + + sleep 2 + done + mark "node-ready-timeout" + return 0 +} + +echo "Fetching OKE bootstrap metadata with a single IMDS request..." + +mark "imds-fetch-start" +INSTANCE_JSON=$(curl -sfL -m 5 --retry 3 \ + -H "Authorization: Bearer Oracle" \ + "http://169.254.169.254/opc/v2/instance") +mark "imds-fetch-end" + +APISERVER=$(echo "$INSTANCE_JSON" | jq -er '.metadata.apiserver_host') +CA_CERT=$(echo "$INSTANCE_JSON" | jq -er '.metadata.cluster_ca_cert') +DNS=$(echo "$INSTANCE_JSON" | jq -r '.metadata.cluster_dns // .metadata.kubedns_svc_ip // "${cluster_dns_ip}"') + +KUBELET_ARGS="\ +--kube-reserved=cpu=70m,memory=1Gi \ +--system-reserved=cpu=100m,memory=256Mi \ +--eviction-hard=memory.available<500Mi" + +echo "Launching OKE installer with explicit bootstrap values..." +mark "oke-installer-start" +bash /etc/oke/oke-install.sh \ + --apiserver-endpoint "$${APISERVER}" \ + --kubelet-ca-cert "$${CA_CERT}" \ + --cluster-dns "$${DNS}" \ + --kubelet-extra-args "$${KUBELET_ARGS}" +mark "oke-installer-end" + +wait_for_kubelet_active & +wait_for_node_ready & + +echo "Bootstrap logic complete." diff --git a/oke-karpenter-fast-start-optimizer-sample/terraform.tfvars.example b/oke-karpenter-fast-start-optimizer-sample/terraform.tfvars.example new file mode 100644 index 0000000..da73209 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/terraform.tfvars.example @@ -0,0 +1,41 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +# Copy this file to terraform.tfvars and set values for your tenancy. +# Do not commit terraform.tfvars. + +config_file_profile = "DEFAULT" +region = "us-ashburn-1" + +tenancy_ocid = "REPLACE_WITH_TENANCY_OCID" +parent_compartment_ocid = "REPLACE_WITH_PARENT_COMPARTMENT_OCID" +parent_compartment_name = "terraform" +project_compartment_ocid = "REPLACE_WITH_PROJECT_COMPARTMENT_OCID" +project_compartment_name = "OKEOptimization" +oke_image_compartment_ocid = "REPLACE_WITH_ORACLE_OKE_IMAGE_COMPARTMENT_OCID" + +ssh_public_key_path = "~/.ssh/id_rsa.pub" +ssh_private_key_path = "~/.ssh/id_rsa" + +# Restrict this to the public IP/CIDR of the workstation or CI runner that will +# SSH to the temporary image-builder VM. +image_builder_ssh_cidr = "203.0.113.10/32" + +# Restrict this before using the cluster outside a short-lived test environment. +allowed_kubernetes_api_cidrs = ["203.0.113.10/32"] + +name = "oke-optimization" + +kubernetes_version = "v1.35.2" +karpenter_chart_version = "1.1.0" +oke_worker_image_display_name = "Oracle-Linux-8.10-2026.02.28-0-OKE-1.35.2-1402" + +karpenter_shape_configs = [ + { + ocpus = 4 + memory_in_gbs = 16 + } +] + +karpenter_node_cpu_limit = "96" +karpenter_node_memory_limit = "384Gi" diff --git a/oke-karpenter-fast-start-optimizer-sample/variables.tf b/oke-karpenter-fast-start-optimizer-sample/variables.tf new file mode 100644 index 0000000..9cfe3c4 --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/variables.tf @@ -0,0 +1,290 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +variable "config_file_profile" { + description = "OCI CLI config profile used by Terraform and Kubernetes exec authentication." + type = string + default = "DEFAULT" +} + +variable "region" { + description = "OCI region for the OKE optimization environment." + type = string + default = "us-ashburn-1" +} + +variable "tenancy_ocid" { + description = "Tenancy OCID." + type = string +} + +variable "parent_compartment_ocid" { + description = "Parent compartment OCID for IAM policies and optional project organization." + type = string +} + +variable "parent_compartment_name" { + description = "Parent compartment name used in IAM policy paths." + type = string + default = "terraform" +} + +variable "project_compartment_ocid" { + description = "Compartment OCID where project resources are created." + type = string +} + +variable "project_compartment_name" { + description = "Project compartment name used in IAM policy statements." + type = string + default = "OKEOptimization" +} + +variable "name" { + description = "Display-name prefix for OCI and Kubernetes resources." + type = string + default = "oke-optimization" +} + +variable "kubernetes_version" { + description = "OKE Kubernetes version. KPO 1.1.0 supports Kubernetes 1.35 and newer." + type = string + default = "v1.35.2" +} + +variable "cluster_type" { + description = "OKE cluster type." + type = string + default = "ENHANCED_CLUSTER" +} + +variable "cni_type" { + description = "OKE CNI. OCI_VCN_IP_NATIVE matches the VCN-native CNI pre-pull optimization in the provided cloud-init." + type = string + default = "OCI_VCN_IP_NATIVE" + + validation { + condition = contains(["OCI_VCN_IP_NATIVE", "FLANNEL_OVERLAY"], var.cni_type) + error_message = "cni_type must be OCI_VCN_IP_NATIVE or FLANNEL_OVERLAY." + } +} + +variable "cluster_dns_ip" { + description = "Cluster DNS service IP passed to optimized OKE worker bootstrap when KPO metadata does not include it." + type = string + default = "10.96.5.5" +} + +variable "oke_image_compartment_ocid" { + description = "Oracle-published OKE image compartment used for OKE worker image lookup in the selected region." + type = string +} + +variable "oke_worker_image_display_name" { + description = "Oracle Linux OKE worker image display name to look up when oke_worker_image_id is null." + type = string + default = "Oracle-Linux-8.10-2026.02.28-0-OKE-1.35.2-1402" +} + +variable "oke_worker_image_id" { + description = "Optional explicit OKE worker image OCID. Leave null to look up oke_worker_image_display_name in the OKE image compartment." + type = string + default = null +} + +variable "ssh_public_key_path" { + description = "SSH public key added to bootstrap and Karpenter-launched worker nodes." + type = string + default = "~/.ssh/id_rsa.pub" +} + +variable "ssh_private_key_path" { + description = "SSH private key used by Terraform to verify the temporary image builder." + type = string + default = "~/.ssh/id_rsa" +} + +variable "allowed_kubernetes_api_cidrs" { + description = "CIDR blocks allowed to reach the public Kubernetes API endpoint. Restrict before production use." + type = list(string) + default = ["0.0.0.0/0"] +} + +variable "vcn_cidr" { + description = "VCN CIDR." + type = string + default = "10.60.0.0/16" +} + +variable "api_endpoint_subnet_cidr" { + description = "Subnet CIDR for the OKE API endpoint." + type = string + default = "10.60.0.0/28" +} + +variable "worker_subnet_cidr" { + description = "Private subnet CIDR for OKE worker primary VNICs." + type = string + default = "10.60.10.0/24" +} + +variable "pod_subnet_cidr" { + description = "Private subnet CIDR for OCI VCN-native pod secondary VNICs." + type = string + default = "10.60.20.0/22" +} + +variable "load_balancer_subnet_cidr" { + description = "Public subnet CIDR for Kubernetes service load balancers." + type = string + default = "10.60.30.0/24" +} + +variable "image_builder_subnet_cidr" { + description = "Public subnet CIDR used only by the temporary optimized OKE image builder." + type = string + default = "10.60.40.0/24" +} + +variable "image_builder_ssh_cidr" { + description = "CIDR allowed to SSH to the temporary optimized OKE image builder." + type = string +} + +variable "image_builder_shape" { + description = "Shape for the temporary optimized OKE image builder." + type = string + default = "VM.Standard.E5.Flex" +} + +variable "image_builder_ocpus" { + description = "OCPUs for the temporary optimized OKE image builder." + type = number + default = 4 +} + +variable "image_builder_memory_in_gbs" { + description = "Memory for the temporary optimized OKE image builder." + type = number + default = 16 +} + +variable "bootstrap_node_shape" { + description = "Managed node pool shape used only to run the KPO controller." + type = string + default = "VM.Standard.E5.Flex" +} + +variable "bootstrap_node_ocpus" { + description = "OCPUs for the bootstrap managed node pool shape." + type = number + default = 4 +} + +variable "bootstrap_node_memory_in_gbs" { + description = "Memory for the bootstrap managed node pool shape." + type = number + default = 16 +} + +variable "bootstrap_node_pool_size" { + description = "Number of managed nodes kept for KPO controller capacity." + type = number + default = 1 +} + +variable "worker_boot_volume_size_in_gbs" { + description = "Boot volume size for bootstrap and Karpenter workers." + type = number + default = 50 +} + +variable "karpenter_chart_version" { + description = "Karpenter Provider OCI Helm chart version." + type = string + default = "1.1.0" +} + +variable "karpenter_namespace" { + description = "Namespace for the KPO controller." + type = string + default = "karpenter" +} + +variable "karpenter_service_account_name" { + description = "Kubernetes service account used by KPO workload identity." + type = string + default = "karpenter" +} + +variable "karpenter_controller_replicas" { + description = "KPO controller replicas. Default is 1 so a single bootstrap node can host the controller." + type = number + default = 1 +} + +variable "karpenter_nodepool_name" { + description = "Name for the Karpenter NodePool and OCINodeClass." + type = string + default = "optimized-ol8" +} + +variable "karpenter_worker_shapes" { + description = "OCI compute shapes Karpenter may launch for optimized workers." + type = list(string) + default = [ + "VM.Standard.E5.Flex" + ] +} + +variable "karpenter_shape_configs" { + description = "Flex shape configs KPO may use for Karpenter-launched workers." + type = list(object({ + ocpus = number + memory_in_gbs = number + baseline_ocpu_utilization = optional(string) + })) + default = [ + { + ocpus = 4 + memory_in_gbs = 16 + } + ] +} + +variable "karpenter_node_cpu_limit" { + description = "Karpenter NodePool CPU limit." + type = string + default = "96" +} + +variable "karpenter_node_memory_limit" { + description = "Karpenter NodePool memory limit." + type = string + default = "384Gi" +} + +variable "karpenter_pod_vnic_ip_count" { + description = "Secondary VNIC IP count for OCI VCN-native Karpenter workers." + type = number + default = 16 +} + +variable "prepull_images" { + description = "Container images pulled in the background before OKE bootstrap starts." + type = list(string) + default = [ + "iad.ocir.io/axoxdievda5j/oke-public-pause@sha256:6e287874898efea60adff9dc1faaee355740b192763f963fe66cf755d995cf32", + "iad.ocir.io/id9y6mi8tcky/oke-public-vcn-native-ip-cni-plugin@sha256:916f9953b84f2788b410c45ba766901e99d9c345d37e2fc35a16c312f27e34f0", + "iad.ocir.io/id9y6mi8tcky/oke-public-kube-proxy@sha256:175559244baa3cbee68adfc1357c18871406c7a2d49dd487d2f04e111e7931a0" + ] +} + +variable "freeform_tags" { + description = "Freeform tags applied to OCI resources." + type = map(string) + default = { + ManagedBy = "terraform" + Project = "OKEOptimization" + } +} diff --git a/oke-karpenter-fast-start-optimizer-sample/versions.tf b/oke-karpenter-fast-start-optimizer-sample/versions.tf new file mode 100644 index 0000000..fe2761e --- /dev/null +++ b/oke-karpenter-fast-start-optimizer-sample/versions.tf @@ -0,0 +1,51 @@ +# Copyright (c) 2024, 2026, Oracle and/or its affiliates. All rights reserved. +# The Universal Permissive License (UPL), Version 1.0 as shown at https://oss.oracle.com/licenses/upl/ + +terraform { + required_version = ">= 1.6.0" + + required_providers { + helm = { + source = "hashicorp/helm" + version = "~> 3.1" + } + kubectl = { + source = "gavinbunney/kubectl" + version = "~> 1.19" + } + oci = { + source = "oracle/oci" + version = "~> 8.13" + } + } +} + +provider "oci" { + config_file_profile = var.config_file_profile + region = var.region +} + +provider "helm" { + kubernetes = { + host = local.kube_host + cluster_ca_certificate = local.kube_ca_certificate + + exec = { + api_version = "client.authentication.k8s.io/v1beta1" + command = "oci" + args = local.kube_exec_args + } + } +} + +provider "kubectl" { + host = local.kube_host + cluster_ca_certificate = local.kube_ca_certificate + load_config_file = false + + exec { + api_version = "client.authentication.k8s.io/v1beta1" + command = "oci" + args = local.kube_exec_args + } +}