Merge branch 'main' into migrate/hami-core-on-pr1480

Author: FouoF

.github/workflows/benchmark-main.yaml

@@ -0,0 +1,71 @@
+# Copyright 2026 NVIDIA CORPORATION
+# SPDX-License-Identifier: Apache-2.0
+
+# Runs benchmarks on every push to main and publishes a per-commit history
+# chart to GitHub Pages at /dev/bench/.
+#
+# Use workflow_dispatch with `gh-pages-branch: gh-pages-test` to dry-run
+# against a throwaway branch before merging changes to this workflow.
+
+name: Benchmark History
+
+on:
+  push:
+    branches:
+      - main
+    paths:
+      - 'pkg/scheduler/**'
+      - 'go.mod'
+      - 'go.sum'
+  workflow_dispatch:
+    inputs:
+      gh-pages-branch:
+        description: 'Branch to publish history to (use a throwaway like gh-pages-test for dry runs)'
+        required: false
+        default: 'gh-pages'
+      auto-push:
+        description: 'Push results to gh-pages-branch'
+        required: false
+        type: boolean
+        default: true
+
+concurrency:
+  group: benchmark-main
+  cancel-in-progress: false
+
+jobs:
+  benchmark:
+    name: Run and publish benchmarks
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+      deployments: write
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-go@v6
+        with:
+          go-version: '1.26.1'
+          cache: true
+
+      - name: Download envtest binaries
+        run: |
+          make envtest
+          echo "KUBEBUILDER_ASSETS=$(bin/setup-envtest use 1.34.0 -p path --bin-dir bin)" >> $GITHUB_ENV
+
+      - name: Run benchmarks
+        run: make benchmark BENCH_OUTPUT=bench.txt
+
+      - name: Publish benchmark history
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: KAI Scheduler
+          tool: go
+          output-file-path: bench.txt
+          gh-pages-branch: ${{ github.event.inputs.gh-pages-branch || 'gh-pages' }}
+          benchmark-data-dir-path: dev/bench
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+          auto-push: ${{ github.event_name == 'push' || github.event.inputs.auto-push == 'true' }}
+          alert-threshold: '150%'
+          fail-on-alert: false
+          comment-on-alert: true

README.md

@@ -134,6 +134,16 @@ Please open a [GitHub issue](https://github.com/kai-scheduler/KAI-scheduler/issu
 
 ---
 
+## Performance Dashboards
+
+KAI Scheduler provides public dashboards for monitoring performance and scale testing:
+
+- **[Scale Tests Dashboard](https://kai-scheduler.github.io/KAI-Scheduler/)**: View historical results from scale tests that validate scheduler performance at large cluster sizes (hundreds to thousands of nodes). Tests run every 24 hours on dedicated infrastructure and measure scheduling performance, topology-aware scheduling, resource allocation, and system stability under load. The dashboard displays execution times, pass/fail status, detailed failure logs, and 30-day historical trends. See [scale tests documentation](docs/developer/scale-tests.md) for technical details.
+
+- **[Benchmarks Dashboard](https://kai-scheduler.github.io/KAI-Scheduler/dev/bench/)**: Track scheduler performance benchmarks across commits to the main branch. The dashboard shows per-commit benchmark history for core scheduler operations, with automatic alerts when performance regresses beyond thresholds.
+
+---
+
 <div align="center">
     <picture>
       <source media="(prefers-color-scheme: dark)" srcset="https://raw.githubusercontent.com/cncf/artwork/refs/heads/main/other/cncf/horizontal/color-whitetext/cncf-color-whitetext.svg">

docs/plugins/reflectjoborder.md

@@ -0,0 +1,83 @@
+# Reflect Job Order Plugin
+
+## Overview
+
+The `reflectjoborder` plugin exposes the order in which the scheduler intends to process pending PodGroups (jobs) during the next scheduling cycle. It is the supported way to answer the question *"where does my workload sit in its queue right now?"*.
+
+The plugin is read-only: it observes the same ordering used by the `Allocate` action and serves it over an HTTP endpoint on the scheduler pod. Each scheduling cycle refreshes the data.
+
+## Enabling the Plugin
+
+The plugin is registered in the scheduler binary but **not enabled by default**. Enable it on the relevant `SchedulingShard`:
+
+```yaml
+apiVersion: kai.scheduler/v1
+kind: SchedulingShard
+metadata:
+  name: default
+spec:
+  plugins:
+    reflectjoborder:
+      enabled: true
+```
+
+When installing via Helm, set `scheduler.plugins` in your values file:
+
+```yaml
+scheduler:
+  plugins:
+    reflectjoborder:
+      enabled: true
+```
+
+See `deployments/kai-scheduler/examples/custom-plugins-actions-values.yaml` for the full plugin-configuration syntax.
+
+## Querying Job Order
+
+The plugin registers an HTTP endpoint `/get-job-order` on the scheduler pod. Port-forward to the scheduler and call it:
+
+```bash
+kubectl port-forward -n kai-scheduler deployment/kai-scheduler-default 8081 &
+sleep 2
+curl -s "localhost:8081/get-job-order" | jq
+```
+
+## Response Format
+
+```json
+{
+  "global_order": [
+    { "id": "team-a/training-7d2", "priority": 100 },
+    { "id": "team-b/inference-9c1", "priority": 75 }
+  ],
+  "queue_order": {
+    "team-a": [
+      { "id": "team-a/training-7d2", "priority": 100 }
+    ],
+    "team-b": [
+      { "id": "team-b/inference-9c1", "priority": 75 }
+    ]
+  }
+}
+```
+
+| Field | Description |
+|---|---|
+| `global_order` | All eligible jobs across all queues, in scheduling order. Index 0 is next to be considered. |
+| `queue_order` | Same jobs grouped by queue. Index within a queue is that workload's position in its queue. |
+| `id` | PodGroup UID (`<namespace>/<name>`). |
+| `priority` | Effective priority used for ordering. |
+
+A workload's queue position is `index in queue_order[<queue>] + 1`.
+
+## Caveats
+
+- **Per-cycle snapshot.** The data is computed at the start of each scheduling cycle; it is not updated continuously. A workload's reported position may be a few seconds stale.
+- **Pending and ready jobs only.** The plugin filters with `FilterNonPending: true` and `FilterUnready: true`, so running jobs and jobs not yet ready for scheduling do not appear (`pkg/scheduler/actions/utils/input_jobs.go`).
+- **Bounded by `QueueDepthPerAction[Allocate]`.** Only the first *N* jobs per queue are reported, where *N* is the configured allocate queue depth. Jobs beyond that depth are excluded, even if pending.
+- **No history.** Only the most recent cycle is exposed; there is no time-series store. To track position over time, scrape the endpoint on an interval.
+- **Not authenticated.** The endpoint is served on the scheduler pod's HTTP port. Reach it via `kubectl port-forward` or an in-cluster client; do not expose it publicly.
+
+## Implementation
+
+Source: `pkg/scheduler/plugins/reflectjoborder/reflect_job_order.go`. The plugin populates `ReflectJobOrder` in `OnSessionOpen` by draining `utils.NewJobsOrderByQueues(...)` and registers `serveJobs` on `/get-job-order`.

pkg/apis/scheduling/v2alpha2/podgroup_webhook.go

@@ -200,10 +200,7 @@ func validateSubGroupMinFields(subGroup *SubGroup, childrenMap map[string][]stri
 				"subgroup %q: minMember cannot be set on a mid-level SubGroup (has child SubGroups); use minSubGroup instead", subGroup.Name)})
 		}
 		if subGroup.MinSubGroup != nil {
-			if *subGroup.MinSubGroup <= 0 {
-				minFieldsErrors = append(minFieldsErrors, &invalidMinSubGroupError{msg: fmt.Sprintf(
-					"subgroup %q: minSubGroup must be greater than 0", subGroup.Name)})
-			} else if int(*subGroup.MinSubGroup) > len(children) {
+			if int(*subGroup.MinSubGroup) > len(children) {
 				minFieldsErrors = append(minFieldsErrors, &minSubGroupExceedsChildCountError{msg: fmt.Sprintf(
 					"subgroup %q: minSubGroup (%d) exceeds the number of direct child SubGroups (%d)",
 					subGroup.Name, *subGroup.MinSubGroup, len(children))})

pkg/apis/scheduling/v2alpha2/podgroup_webhook_test.go

@@ -171,15 +171,6 @@ func TestValidateSubGroups(t *testing.T) {
 			},
 			wantErr: nil,
 		},
-		{
-			name: "Invalid: minSubGroup = 0 on SubGroup with children",
-			subGroups: []SubGroup{
-				{Name: "parent", MinSubGroup: ptr.To(int32(0))},
-				{Name: "child-1", Parent: ptr.To("parent"), MinMember: ptr.To(int32(4))},
-				{Name: "child-2", Parent: ptr.To("parent"), MinMember: ptr.To(int32(4))},
-			},
-			wantErr: &validationErrors{minDefinitionErrors: []error{&invalidMinSubGroupError{msg: `subgroup "parent": minSubGroup must be greater than 0`}}},
-		},
 		{
 			name: "Invalid: minSubGroup = 0 on leaf SubGroup",
 			subGroups: []SubGroup{
@@ -324,17 +315,6 @@ func TestValidatePodGroupSpec(t *testing.T) {
 			},
 			want: &validationErrors{minDefinitionErrors: []error{&minSubGroupExceedsChildCountError{msg: "minSubGroup (1) exceeds the number of direct child SubGroups (0)"}}},
 		},
-		{
-			name: "Invalid: minSubGroup = 0 on PodGroup",
-			spec: PodGroupSpec{
-				MinSubGroup: ptr.To(int32(0)),
-				SubGroups: []SubGroup{
-					{Name: "a", MinMember: ptr.To(int32(4))},
-					{Name: "b", MinMember: ptr.To(int32(4))},
-				},
-			},
-			want: &validationErrors{minDefinitionErrors: []error{&invalidMinSubGroupError{msg: "minSubGroup at the podgroup level must be equal to or greater than 1"}}},
-		},
 	}
 
 	for _, tt := range tests {

pkg/scheduler/api/podgroup_info/allocation_info_test.go

@@ -220,6 +220,79 @@ func Test_GetTasksToAllocate(t *testing.T) {
 	}
 }
 
+func Test_GetTasksToAllocate_MinSubGroupZero(t *testing.T) {
+	tests := []struct {
+		name          string
+		subGroupTasks map[string][]*pod_info.PodInfo
+		minAvailMap   map[string]int32
+		wantNumTasks  int
+	}{
+		{
+			name: "root minSubGroup=0, all children unsatisfied: gang skipped, elastic returns one child's tasks",
+			subGroupTasks: map[string][]*pod_info.PodInfo{
+				"sgA": {
+					simpleTask("taskA1", "sgA", pod_status.Pending),
+				},
+				"sgB": {
+					simpleTask("taskB1", "sgB", pod_status.Pending),
+				},
+			},
+			minAvailMap:  map[string]int32{"sgA": 1, "sgB": 1},
+			wantNumTasks: 1,
+		},
+		{
+			name: "root minSubGroup=0, all children satisfied: elastic returns no tasks",
+			subGroupTasks: map[string][]*pod_info.PodInfo{
+				"sgA": {
+					simpleTask("taskA1", "sgA", pod_status.Running),
+				},
+				"sgB": {
+					simpleTask("taskB1", "sgB", pod_status.Running),
+				},
+			},
+			minAvailMap:  map[string]int32{"sgA": 1, "sgB": 1},
+			wantNumTasks: 0,
+		},
+		{
+			name: "root minSubGroup=0, one satisfied + one with elastic surplus: returns one elastic task",
+			subGroupTasks: map[string][]*pod_info.PodInfo{
+				"sgA": {
+					simpleTask("taskA1", "sgA", pod_status.Running),
+				},
+				"sgB": {
+					simpleTask("taskB1", "sgB", pod_status.Running),
+					simpleTask("taskB2", "sgB", pod_status.Pending),
+				},
+			},
+			minAvailMap:  map[string]int32{"sgA": 1, "sgB": 1},
+			wantNumTasks: 1,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			pg := NewPodGroupInfo("pg")
+			root := subgroup_info.NewSubGroupSet(subgroup_info.RootSubGroupSetName, nil)
+			min := int32(0)
+			root.SetMinSubGroup(&min)
+			pg.RootSubGroupSet = root
+			pg.PodSets = make(map[string]*subgroup_info.PodSet)
+			for subGroupName, pods := range tt.subGroupTasks {
+				ps := subgroup_info.NewPodSet(subGroupName, tt.minAvailMap[subGroupName], nil)
+				root.AddPodSet(ps)
+				pg.PodSets[subGroupName] = ps
+				for _, pod := range pods {
+					pg.AddTaskInfo(pod)
+				}
+			}
+			gotTasks := GetTasksToAllocate(pg, subGroupOrderFn, tasksOrderFn, true)
+			if len(gotTasks) != tt.wantNumTasks {
+				t.Errorf("GetTasksToAllocate len = %d, want %d", len(gotTasks), tt.wantNumTasks)
+			}
+		})
+	}
+}
+
 func Test_GetTasksToAllocateRequestedGPUs(t *testing.T) {
 	pg := NewPodGroupInfo("test-podgroup")
 	pg.GetAllPodSets()[DefaultSubGroup].SetMinAvailable(1)

pkg/scheduler/api/podgroup_info/eviction_info_test.go

@@ -324,6 +324,78 @@ func TestGetTasksToEvict_HierarchicalTree(t *testing.T) {
 			expectedHasMoreTasks: true,
 			numExpectTasks:       2,
 		},
+		{
+			name: "MinSubGroupZero_RootEntireSubtreeElasticDroppable",
+			job: func() *PodGroupInfo {
+				// Root with minSubGroup=0 and one PodSet child at exactly min.
+				// Phase 2 fires because 0 < numSatisfied(1) → drop the whole child.
+				psA := subgroup_info.NewPodSet("ps-a", 1, nil)
+				psA.AssignTask(simpleTask("pod-1", "ps-a", pod_status.Running))
+
+				root := subgroup_info.NewSubGroupSet(subgroup_info.RootSubGroupSetName, nil)
+				root.SetMinSubGroup(ptr.To(int32(0)))
+				root.AddPodSet(psA)
+
+				return &PodGroupInfo{
+					RootSubGroupSet: root,
+					PodSets:         root.GetDescendantPodSets(),
+				}
+			}(),
+			expectedHasMoreTasks: false,
+			numExpectTasks:       1,
+		},
+		{
+			name: "MinSubGroupZero_NestedInnerSubtreeElasticDroppable",
+			job: func() *PodGroupInfo {
+				// Inner SubGroupSet with minSubGroup=0 containing a PodSet at min.
+				// Root requires inner (no minSubGroup). Inner is treated as fully elastic:
+				// phase 1 finds elastic surplus inside inner → recurses → inner phase 2 drops ps-a.
+				psA := subgroup_info.NewPodSet("ps-a", 1, nil)
+				psA.AssignTask(simpleTask("pod-1", "ps-a", pod_status.Running))
+
+				inner := subgroup_info.NewSubGroupSet("inner", nil)
+				inner.SetMinSubGroup(ptr.To(int32(0)))
+				inner.AddPodSet(psA)
+
+				root := subgroup_info.NewSubGroupSet(subgroup_info.RootSubGroupSetName, nil)
+				root.AddSubGroup(inner)
+
+				return &PodGroupInfo{
+					RootSubGroupSet: root,
+					PodSets:         root.GetDescendantPodSets(),
+				}
+			}(),
+			expectedHasMoreTasks: false,
+			numExpectTasks:       1,
+		},
+		{
+			name: "MinSubGroupZero_NestedInnerLeavesSiblingAlone",
+			job: func() *PodGroupInfo {
+				// Root requires both children; inner has minSubGroup=0 (fully elastic),
+				// sibling required ps-keep is at min. Eviction must come from inner only.
+				psElastic := subgroup_info.NewPodSet("ps-elastic", 1, nil)
+				psElastic.AssignTask(simpleTask("pod-1", "ps-elastic", pod_status.Running))
+
+				inner := subgroup_info.NewSubGroupSet("inner", nil)
+				inner.SetMinSubGroup(ptr.To(int32(0)))
+				inner.AddPodSet(psElastic)
+
+				psKeep := subgroup_info.NewPodSet("ps-keep", 1, nil)
+				psKeep.AssignTask(simpleTask("pod-2", "ps-keep", pod_status.Running))
+
+				root := subgroup_info.NewSubGroupSet(subgroup_info.RootSubGroupSetName, nil)
+				root.AddSubGroup(inner)
+				root.AddPodSet(psKeep)
+
+				return &PodGroupInfo{
+					RootSubGroupSet: root,
+					PodSets:         root.GetDescendantPodSets(),
+				}
+			}(),
+			expectedHasMoreTasks: true, // pod-2 in ps-keep stays
+			numExpectTasks:       1,
+			expectedTaskNames:    []string{"pod-1"},
+		},
 	}
 
 	for _, tt := range tests {