diff --git a/cmd/ghalistener/metrics/metrics.go b/cmd/ghalistener/metrics/metrics.go index a9353cccbd..2280c78894 100644 --- a/cmd/ghalistener/metrics/metrics.go +++ b/cmd/ghalistener/metrics/metrics.go @@ -45,6 +45,7 @@ const ( MetricIdleRunners = "gha_idle_runners" MetricStartedJobsTotal = "gha_started_jobs_total" MetricCompletedJobsTotal = "gha_completed_jobs_total" + MetricJobQueueDurationSeconds = "gha_job_queue_duration_seconds" MetricJobStartupDurationSeconds = "gha_job_startup_duration_seconds" MetricJobExecutionDurationSeconds = "gha_job_execution_duration_seconds" ) @@ -71,6 +72,7 @@ var metricsHelp = metricsHelpRegistry{ MetricIdleRunners: "Number of registered runners not running a job.", }, histograms: map[string]string{ + MetricJobQueueDurationSeconds: "Time spent waiting for workflow jobs to get assigned to the scale set after queueing (in seconds).", MetricJobStartupDurationSeconds: "Time spent waiting for workflow job to get started on the runner owned by the scale set (in seconds).", MetricJobExecutionDurationSeconds: "Time spent executing workflow jobs by the scale set (in seconds).", }, @@ -259,6 +261,16 @@ var defaultMetrics = v1alpha1.MetricsConfig{ }, }, Histograms: map[string]*v1alpha1.HistogramMetric{ + MetricJobQueueDurationSeconds: { + Labels: []string{ + labelKeyEnterprise, + labelKeyOrganization, + labelKeyRepository, + labelKeyJobName, + labelKeyEventName, + }, + Buckets: defaultRuntimeBuckets, + }, MetricJobStartupDurationSeconds: { Labels: []string{ labelKeyEnterprise, @@ -481,6 +493,9 @@ func (e *exporter) PublishJobStarted(msg *actions.JobStarted) { l := e.startedJobLabels(msg) e.incCounter(MetricStartedJobsTotal, l) + queueDuration := msg.ScaleSetAssignTime.Unix() - msg.QueueTime.Unix() + e.observeHistogram(MetricJobQueueDurationSeconds, l, float64(queueDuration)) + startupDuration := msg.RunnerAssignTime.Unix() - msg.ScaleSetAssignTime.Unix() e.observeHistogram(MetricJobStartupDurationSeconds, l, float64(startupDuration)) } diff --git a/cmd/ghalistener/metrics/metrics_test.go b/cmd/ghalistener/metrics/metrics_test.go index 850560fbc1..dbf262ec8f 100644 --- a/cmd/ghalistener/metrics/metrics_test.go +++ b/cmd/ghalistener/metrics/metrics_test.go @@ -67,6 +67,10 @@ func TestInstallMetrics(t *testing.T) { Buckets: []float64{0.1, 1}, }, // histogram metric should be registered with default runtime buckets + MetricJobQueueDurationSeconds: { + Labels: []string{labelKeyRepository}, + }, + // histogram metric should be registered with default runtime buckets MetricJobStartupDurationSeconds: { Labels: []string{labelKeyRepository}, }, @@ -77,7 +81,7 @@ func TestInstallMetrics(t *testing.T) { got := installMetrics(metricsConfig, reg, logr.Discard()) assert.Len(t, got.counters, 1) assert.Len(t, got.gauges, 1) - assert.Len(t, got.histograms, 2) + assert.Len(t, got.histograms, 3) assert.Equal(t, got.counters[MetricStartedJobsTotal].config, metricsConfig.Counters[MetricStartedJobsTotal]) assert.Equal(t, got.gauges[MetricAssignedJobs].config, metricsConfig.Gauges[MetricAssignedJobs])