Add a metric of AppWrappers counts per phase, step and priority (#52)

* Add appWrappersCount metric

* Make go-mod-tidy happy

Author: ronensc

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 	github.com/onsi/ginkgo/v2 v2.11.0
 	github.com/onsi/gomega v1.27.10
+	github.com/prometheus/client_golang v1.16.0
 	gopkg.in/inf.v0 v0.9.1
 	k8s.io/api v0.28.3
 	k8s.io/apimachinery v0.28.3
@@ -42,7 +43,6 @@ require (
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/prometheus/client_golang v1.16.0 // indirect
 	github.com/prometheus/client_model v0.4.0 // indirect
 	github.com/prometheus/common v0.44.0 // indirect
 	github.com/prometheus/procfs v0.10.1 // indirect

internal/controller/dispatch_logic.go

@@ -19,13 +19,15 @@ package controller
 import (
 	"context"
 	"sort"
+	"strconv"
 	"time"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	mcadv1beta1 "github.com/project-codeflare/mcad/api/v1beta1"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // Compute available cluster capacity
@@ -78,12 +80,21 @@ func (r *AppWrapperReconciler) listAppWrappers(ctx context.Context) (map[int]Wei
 	}
 	requests := map[int]Weights{}        // total request per priority level
 	queue := []*mcadv1beta1.AppWrapper{} // queued appWrappers
+
+	appWrapperCount := map[phaseStepPriority]int{}
+
 	for _, appWrapper := range appWrappers.Items {
 		// get phase from cache if available as reconciler cache may be lagging
 		phase, step := r.getCachedPhase(&appWrapper)
+		priority := int(appWrapper.Spec.Priority)
+		key := phaseStepPriority{phase, step, priority}
+		if _, exists := appWrapperCount[key]; !exists {
+			appWrapperCount[key] = 0
+		}
+		appWrapperCount[key]++
 		// make sure to initialize weights for every known priority level
-		if requests[int(appWrapper.Spec.Priority)] == nil {
-			requests[int(appWrapper.Spec.Priority)] = Weights{}
+		if requests[priority] == nil {
+			requests[priority] = Weights{}
 		}
 		if step != mcadv1beta1.Idle {
 			// use max request among AppWrapper request and total request of non-terminated AppWrapper pods
@@ -109,6 +120,13 @@ func (r *AppWrapperReconciler) listAppWrappers(ctx context.Context) (map[int]Wei
 			queue = append(queue, &copy)
 		}
 	}
+	// update AppWrapper count metric
+	appWrappersCount.Reset()
+	for key, count := range appWrapperCount {
+		appWrappersCount.With(
+			prometheus.Labels{"phase": string(key.phase), "step": string(key.step), "priority": strconv.Itoa(key.priority)},
+		).Set(float64(count))
+	}
 	// propagate reservations at all priority levels to all levels below
 	assertPriorities(requests)
 	// order AppWrapper queue based on priority and precedence (creation time)

internal/controller/metrics.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2023 IBM Corporation.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	mcadv1beta1 "github.com/project-codeflare/mcad/api/v1beta1"
+	"github.com/prometheus/client_golang/prometheus"
+	"sigs.k8s.io/controller-runtime/pkg/metrics"
+)
+
+type phaseStepPriority struct {
+	phase    mcadv1beta1.AppWrapperPhase
+	step     mcadv1beta1.AppWrapperStep
+	priority int
+}
+
+var (
+	appWrappersCount = prometheus.NewGaugeVec(prometheus.GaugeOpts{
+		Subsystem: "mcad",
+		Name:      "appwrappers_count",
+		Help:      "AppWrappers count per phase, step and priority",
+	}, []string{"phase", "step", "priority"})
+)
+
+func init() {
+	metrics.Registry.MustRegister(
+		appWrappersCount,
+	)
+}