feat: integrate secret resolution into HTTP prober and agent components

This PR integrates the secret infrastructure from PR #1468 into the HTTP prober and throughout the agent:

- HTTP Prober Integration:
  - Updated HTTP prober to use secret resolution for authentication
  - Enhanced multihttp script support with secret interpolation
  - Comprehensive test coverage for secret scenarios

- Agent Integration:
  - Updated main.go to wire up secret providers
  - Enhanced adhoc, checks, and scraper components with secret support
  - Updated all test files to handle new secret provider interfaces

- Test Updates:
  - All prober test files updated to implement new SecretProvider interface
  - Browser, scripted, and multihttp tests enhanced for secret scenarios
  - Comprehensive test coverage for secret resolution workflows

This completes the secret store support feature by integrating the core infrastructure into all agent components.

Author: d0ugal

cmd/synthetic-monitoring-agent/main.go

@@ -314,7 +314,12 @@ func run(args []string, stdout io.Writer) error {
 
 	publisher := publisherFactory(ctx, tm, zl.With().Str("subsystem", "publisher").Str("version", config.SelectedPublisher).Logger(), promRegisterer)
 	limits := limits.NewTenantLimits(tm)
-	secrets := secrets.NewTenantSecrets(tm, zl.With().Str("subsystem", "secretstore").Logger())
+
+	// Create secret provider with caching (60-second TTL)
+	secretProvider := secrets.NewSecretProvider(tm, 60*time.Second, zl.With().Str("subsystem", "secretstore").Logger())
+
+	// Wrap with capability awareness - capabilities will be set after probe registration
+	capabilityAwareSecretProvider := secrets.NewUpdatableCapabilityAwareSecretProvider(secretProvider)
 
 	telemetry := telemetry.NewTelemeter(
 		ctx, uuid.New().String(), time.Duration(config.TelemetryTimeSpan)*time.Minute,
@@ -335,7 +340,7 @@ func run(args []string, stdout io.Writer) error {
 		K6Runner:       k6Runner,
 		ScraperFactory: scraper.New,
 		TenantLimits:   limits,
-		TenantSecrets:  secrets,
+		TenantSecrets:  capabilityAwareSecretProvider,
 		Telemeter:      telemetry,
 		UsageReporter:  usageReporter,
 	})
@@ -356,7 +361,7 @@ func run(args []string, stdout io.Writer) error {
 		PromRegisterer: promRegisterer,
 		Features:       features,
 		K6Runner:       k6Runner,
-		TenantSecrets:  secrets,
+		TenantSecrets:  capabilityAwareSecretProvider,
 	})
 	if err != nil {
 		return fmt.Errorf("cannot create ad-hoc checks handler: %w", err)

internal/adhoc/adhoc.go

@@ -8,10 +8,8 @@ import (
 	"io"
 	"time"
 
-	"github.com/grafana/synthetic-monitoring-agent/internal/secrets"
-
 	"github.com/prometheus/client_golang/prometheus"
-	"github.com/prometheus/prometheus/prompb"
+	prompb "github.com/prometheus/prometheus/prompb"
 	"github.com/rs/zerolog"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/codes"
@@ -24,6 +22,7 @@ import (
 	"github.com/grafana/synthetic-monitoring-agent/internal/model"
 	"github.com/grafana/synthetic-monitoring-agent/internal/prober"
 	"github.com/grafana/synthetic-monitoring-agent/internal/pusher"
+	"github.com/grafana/synthetic-monitoring-agent/internal/secrets"
 	"github.com/grafana/synthetic-monitoring-agent/internal/version"
 	sm "github.com/grafana/synthetic-monitoring-agent/pkg/pb/synthetic_monitoring"
 )
@@ -112,7 +111,7 @@ type HandlerOpts struct {
 	PromRegisterer prometheus.Registerer
 	Features       feature.Collection
 	K6Runner       k6runner.Runner
-	TenantSecrets  *secrets.TenantSecrets
+	TenantSecrets  secrets.SecretProvider
 
 	// these two fields exists so that tests can pass alternate
 	// implementations, they are unexported so that clients of this

internal/adhoc/adhoc_test.go

@@ -366,6 +366,10 @@ func (s testSecretStore) GetSecretCredentials(ctx context.Context, tenantId mode
 	}, nil
 }
 
+func (s testSecretStore) GetSecretValue(ctx context.Context, tenantID model.GlobalID, secretKey string) (string, error) {
+	return "", nil
+}
+
 func TestDefaultRunnerFactory(t *testing.T) {
 	t.Parallel()
 

internal/checks/checks.go

@@ -13,6 +13,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/jpillora/backoff"
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/rs/zerolog"
@@ -22,7 +23,6 @@ import (
 	"google.golang.org/grpc/status"
 
 	logproto "github.com/grafana/loki/pkg/push"
-
 	"github.com/grafana/synthetic-monitoring-agent/internal/error_types"
 	"github.com/grafana/synthetic-monitoring-agent/internal/feature"
 	"github.com/grafana/synthetic-monitoring-agent/internal/k6runner"
@@ -81,7 +81,7 @@ type Updater struct {
 	k6Runner       k6runner.Runner
 	scraperFactory scraper.Factory
 	tenantLimits   *limits.TenantLimits
-	tenantSecrets  *secrets.TenantSecrets
+	tenantSecrets  secrets.SecretProvider
 	telemeter      *telemetry.Telemeter
 	usageReporter  usage.Reporter
 }
@@ -108,7 +108,7 @@ type (
 type UpdaterOptions struct {
 	Conn           *grpc.ClientConn
 	Logger         zerolog.Logger
-	Backoff        Backoffer
+	Backoff        *backoff.Backoff
 	Publisher      pusher.Publisher
 	TenantCh       chan<- sm.Tenant
 	IsConnected    func(bool)
@@ -117,8 +117,8 @@ type UpdaterOptions struct {
 	K6Runner       k6runner.Runner
 	ScraperFactory scraper.Factory
 	TenantLimits   *limits.TenantLimits
+	TenantSecrets  secrets.SecretProvider
 	Telemeter      *telemetry.Telemeter
-	TenantSecrets  *secrets.TenantSecrets
 	UsageReporter  usage.Reporter
 }
 
@@ -243,6 +243,7 @@ func NewUpdater(opts UpdaterOptions) (*Updater, error) {
 		tenantLimits:   opts.TenantLimits,
 		tenantSecrets:  opts.TenantSecrets,
 		telemeter:      opts.Telemeter,
+		usageReporter:  opts.UsageReporter,
 		metrics: metrics{
 			changeErrorsCounter: changeErrorsCounter,
 			changesCounter:      changesCounter,
@@ -252,7 +253,6 @@ func NewUpdater(opts UpdaterOptions) (*Updater, error) {
 			scrapeErrorCounter:  scrapeErrorCounter,
 			scrapesCounter:      scrapesCounter,
 		},
-		usageReporter: opts.UsageReporter,
 	}, nil
 }
 
@@ -332,7 +332,6 @@ func handleError(ctx context.Context, logger zerolog.Logger, backoff Backoffer,
 	return false, nil
 }
 
-//goland:noinspection GoBoolExpressions
 func (c *Updater) loop(ctx context.Context) (bool, error) {
 	connected := false
 
@@ -399,9 +398,23 @@ func (c *Updater) loop(ctx context.Context) (bool, error) {
 
 	c.probe = &result.Probe
 
-	err = c.usageReporter.ReportProbe(ctx, result.Probe, c.features)
-	if err != nil {
-		c.logger.Warn().Err(err).Msg("reporting usage failed")
+	// Report usage if reporter is configured
+	if c.usageReporter != nil {
+		err := c.usageReporter.ReportProbe(ctx, result.Probe, c.features)
+		if err != nil {
+			c.logger.Warn().Err(err).Msg("reporting usage failed")
+		}
+	}
+
+	// Update secret provider with probe capabilities if it supports it
+	if updatableSecretProvider, ok := c.tenantSecrets.(secrets.UpdatableCapabilityAwareSecretProvider); ok {
+		updatableSecretProvider.UpdateCapabilities(c.probe.Capabilities)
+		logger := c.logger.With().Int64("probe_id", c.probe.Id).Logger()
+		enableProtocolSecrets := false
+		if c.probe.Capabilities != nil {
+			enableProtocolSecrets = c.probe.Capabilities.EnableProtocolSecrets
+		}
+		logger.Debug().Bool("enable_protocol_secrets", enableProtocolSecrets).Msg("updated secret provider with probe capabilities")
 	}
 
 	logger := c.logger.With().Int64("probe_id", c.probe.Id).Logger()

internal/checks/checks_test.go

@@ -473,7 +473,7 @@ func testScraperFactory(ctx context.Context, check model.Check, publisher pusher
 	k6Runner k6runner.Runner,
 	labelsLimiter scraper.LabelsLimiter,
 	telemeter *telemetry.Telemeter,
-	secretStore *secrets.TenantSecrets,
+	secretStore secrets.SecretProvider,
 ) (*scraper.Scraper, error) {
 	return scraper.NewWithOpts(
 		ctx,

internal/prober/browser/browser_test.go

@@ -89,3 +89,7 @@ type noopSecretStore struct{}
 func (n noopSecretStore) GetSecretCredentials(ctx context.Context, tenantID model.GlobalID) (*sm.SecretStore, error) {
 	return &sm.SecretStore{}, nil
 }
+
+func (n noopSecretStore) GetSecretValue(ctx context.Context, tenantID model.GlobalID, secretKey string) (string, error) {
+	return "", nil
+}