fix(e2e): webhook-driven, auto-versioned CMT + fix non-PR dispatch 422

config/config-matterwick.default.json

@@ -83,5 +83,7 @@
   "E2EReleasePatternPrefix": "release-",
   "E2ENightlyTriggerWorkflowName": "E2E Nightly Trigger",
   "E2ETestWorkflowNames": ["Electron Playwright Tests", "E2E", "Compatibility Matrix Testing"],
-  "E2EInstanceMaxAge": 6
+  "E2EInstanceMaxAge": 6,
+  "CMTTriggerWorkflowName": "CMT Provisioner",
+  "CMTServerVersions": ["10.11.0", "11.7.0"]
 }

server/config.go

@@ -123,6 +123,32 @@ type MatterwickConfig struct {
 	// Set to the longest expected E2E run duration plus a small buffer.
 	// Default (0): 3 hours.
 	E2EInstanceMaxAge int
+
+	// CMTTriggerWorkflowName is the workflow name (the "name:" field) of the lightweight,
+	// scheduled CMT trigger workflow in the desktop/mobile repos. When matterwick receives
+	// a workflow_run "requested" event for this workflow it provisions one instance per
+	// version in CMTServerVersions and dispatches compatibility-matrix-testing.yml.
+	CMTTriggerWorkflowName string
+
+	// CMTServerVersions is the hardcoded set of Mattermost server versions CMT runs
+	// against (e.g. the active ESR plus the current feature release). Values must be valid
+	// Mattermost image tags (full semver, no "v" prefix, e.g. "10.11.0"). Overridable via
+	// the gitops config; when empty matterwick falls back to defaultCMTServerVersions.
+	CMTServerVersions []string
+}
+
+// defaultCMTServerVersions is the fallback CMT version set used when Config.CMTServerVersions
+// is empty. Mattermost actively supports v11.x feature releases and the v10.11 ESR, so the
+// default covers the current ESR line plus a v11 release. Update as ESR/feature lines change.
+var defaultCMTServerVersions = []string{"10.11.0", "11.7.0"}
+
+// CMTVersions returns the configured CMT server versions, or the hardcoded default set when
+// none are configured.
+func (c *MatterwickConfig) CMTVersions() []string {
+	if len(c.CMTServerVersions) > 0 {
+		return c.CMTServerVersions
+	}
+	return defaultCMTServerVersions
 }
 
 func findConfigFile(fileName string) string {

server/e2e_tests.go

@@ -1048,11 +1048,11 @@ func (s *Server) buildInstanceDetailsJSON(instances []*E2EInstance) (string, err
 	return string(jsonBytes), nil
 }
 
-// dispatchDesktopE2EWorkflow triggers the desktop E2E workflow via GitHub Actions API.
-// trackingKey is the s.e2eInstances map key for this run; when non-empty it is passed
-// as the "mw_tracking_key" workflow input so the workflow_run completed handler can do
-// a direct key lookup instead of fragile SHA suffix matching.
-func (s *Server) dispatchDesktopE2EWorkflow(repoOwner, repoName, ref, sha, instanceDetailsJSON, runType, trackingKey string, nightly bool) error {
+// dispatchDesktopE2EWorkflow triggers the desktop E2E workflow (e2e-functional.yml) via the
+// GitHub Actions API. Cleanup is driven by the workflow_run completed event matched on the
+// commit SHA, so no tracking key is passed as an input (e2e-functional.yml does not declare
+// one, and GitHub rejects a workflow_dispatch carrying an undeclared input with a 422).
+func (s *Server) dispatchDesktopE2EWorkflow(repoOwner, repoName, ref, sha, instanceDetailsJSON, runType string, nightly bool) error {
 	ctx := context.Background()
 	client := newGithubClient(s.Config.GithubAccessToken)
 
@@ -1085,9 +1085,6 @@ func (s *Server) dispatchDesktopE2EWorkflow(repoOwner, repoName, ref, sha, insta
 		"run_type":          runType,
 		"nightly":           fmt.Sprintf("%t", nightly),
 	}
-	if trackingKey != "" {
-		workflowInputs["mw_tracking_key"] = trackingKey
-	}
 
 	// Use REST API to trigger workflow dispatch (v32 go-github compatibility)
 	req, err := client.NewRequest("POST",
@@ -1116,11 +1113,11 @@ func (s *Server) dispatchDesktopE2EWorkflow(repoOwner, repoName, ref, sha, insta
 	return nil
 }
 
-// dispatchMobileE2EWorkflow triggers the mobile E2E workflow via GitHub Actions API.
-// trackingKey is the s.e2eInstances map key for this run; when non-empty it is passed
-// as the "mw_tracking_key" workflow input so the workflow_run completed handler can do
-// a direct key lookup instead of fragile SHA suffix matching.
-func (s *Server) dispatchMobileE2EWorkflow(repoOwner, repoName, ref, sha, site1URL, site2URL, site3URL, platform, runType, trackingKey string) error {
+// dispatchMobileE2EWorkflow triggers the mobile E2E workflow (e2e-detox-pr.yml) via the
+// GitHub Actions API. Cleanup is driven by the workflow_run completed event matched on the
+// commit SHA, so no tracking key is passed as an input (e2e-detox-pr.yml does not declare
+// one, and GitHub rejects a workflow_dispatch carrying an undeclared input with a 422).
+func (s *Server) dispatchMobileE2EWorkflow(repoOwner, repoName, ref, sha, site1URL, site2URL, site3URL, platform, runType string) error {
 	ctx := context.Background()
 	client := newGithubClient(s.Config.GithubAccessToken)
 
@@ -1138,9 +1135,6 @@ func (s *Server) dispatchMobileE2EWorkflow(repoOwner, repoName, ref, sha, site1U
 		"PLATFORM":       platform,
 		"run_type":       runType,
 	}
-	if trackingKey != "" {
-		workflowInputs["mw_tracking_key"] = trackingKey
-	}
 
 	// Use REST API to trigger workflow dispatch (v32 go-github compatibility)
 	req, err := client.NewRequest("POST",

server/push_events.go

@@ -140,7 +140,7 @@ func (s *Server) handlePushEventE2E(event *github.PushEvent, branch string) {
 	s.e2eInstances[key] = instances
 	s.e2eInstancesLock.Unlock()
 
-	err = s.triggerE2EWorkflowForPushEvent(repoName, instanceType, branch, sha, key, instances)
+	err = s.triggerE2EWorkflowForPushEvent(repoName, instanceType, branch, sha, instances)
 	if err != nil {
 		logger.WithError(err).Error("Failed to trigger E2E workflow")
 		s.e2eInstancesLock.Lock()
@@ -249,8 +249,8 @@ func getRunnerForPlatform(platform string) string {
 }
 
 // triggerE2EWorkflowForPushEvent routes to the desktop or mobile dispatch function.
-// trackingKey is embedded in workflow inputs as mw_tracking_key for cleanup on completion.
-func (s *Server) triggerE2EWorkflowForPushEvent(repoName, instanceType, branch, sha, trackingKey string, instances []*E2EInstance) error {
+// Cleanup is driven by the workflow_run completed event matched on the commit SHA.
+func (s *Server) triggerE2EWorkflowForPushEvent(repoName, instanceType, branch, sha string, instances []*E2EInstance) error {
 	logger := s.Logger.WithFields(logrus.Fields{
 		"repo":         repoName,
 		"instanceType": instanceType,
@@ -265,14 +265,14 @@ func (s *Server) triggerE2EWorkflowForPushEvent(repoName, instanceType, branch,
 	}
 
 	if instanceType == "desktop" {
-		return s.triggerDesktopE2EWorkflowForPushEvent(repoOwner, repoName, branch, sha, trackingKey, instances)
+		return s.triggerDesktopE2EWorkflowForPushEvent(repoOwner, repoName, branch, sha, instances)
 	}
 
-	return s.triggerMobileE2EWorkflowForPushEvent(repoOwner, repoName, branch, sha, trackingKey, instances)
+	return s.triggerMobileE2EWorkflowForPushEvent(repoOwner, repoName, branch, sha, instances)
 }
 
 // triggerDesktopE2EWorkflowForPushEvent dispatches the desktop E2E workflow.
-func (s *Server) triggerDesktopE2EWorkflowForPushEvent(repoOwner, repoName, branch, sha, trackingKey string, instances []*E2EInstance) error {
+func (s *Server) triggerDesktopE2EWorkflowForPushEvent(repoOwner, repoName, branch, sha string, instances []*E2EInstance) error {
 	logger := s.Logger.WithFields(logrus.Fields{
 		"repo":   repoName,
 		"branch": branch,
@@ -291,11 +291,11 @@ func (s *Server) triggerDesktopE2EWorkflowForPushEvent(repoOwner, repoName, bran
 		runType = "RELEASE"
 	}
 
-	return s.dispatchDesktopE2EWorkflow(repoOwner, repoName, branch, sha, instanceDetailsJSON, runType, trackingKey, false)
+	return s.dispatchDesktopE2EWorkflow(repoOwner, repoName, branch, sha, instanceDetailsJSON, runType, false)
 }
 
 // triggerMobileE2EWorkflowForPushEvent dispatches the mobile E2E workflow (e2e-detox-pr.yml).
-func (s *Server) triggerMobileE2EWorkflowForPushEvent(repoOwner, repoName, branch, sha, trackingKey string, instances []*E2EInstance) error {
+func (s *Server) triggerMobileE2EWorkflowForPushEvent(repoOwner, repoName, branch, sha string, instances []*E2EInstance) error {
 	logger := s.Logger.WithFields(logrus.Fields{
 		"repo":   repoName,
 		"branch": branch,
@@ -321,6 +321,6 @@ func (s *Server) triggerMobileE2EWorkflowForPushEvent(repoOwner, repoName, branc
 		repoOwner, repoName, branch, sha,
 		instances[0].URL, instances[1].URL, instances[2].URL,
 		"both", // push events always test both iOS and Android
-		runType, trackingKey,
+		runType,
 	)
 }

server/workflow_run.go

@@ -78,39 +78,18 @@ func (s *Server) handleWorkflowRunEventWithInputs(payload *WorkflowRunWebhookPay
 		"head_sha": headSHA,
 	})
 
-	// CMT: "CMT Provisioner" (user-dispatched) provisions servers; "Compatibility Matrix Testing" runs tests.
-	if strings.Contains(workflowName, "cmt") || strings.Contains(workflowName, "CMT") {
-		if payload.Action == "completed" {
-			logger.Debug("CMT trigger workflow completed; sha-based cleanup is primary")
-			s.handleCMTRunCleanup(repoName, headSHA, logger)
-			return
-		}
-		if payload.Action != "requested" {
-			logger.Debug("Ignoring CMT workflow action (not requested or completed)")
-			return
-		}
-		logger.Info("Processing CMT workflow_run event")
-		serverVersionsStr, ok := payload.WorkflowRun.Inputs["server_versions"]
-		if !ok || serverVersionsStr == "" {
-			logger.Error("No server_versions found in workflow inputs")
-			return
-		}
-		serverVersions := parseServerVersionsFromString(serverVersionsStr)
-		if len(serverVersions) == 0 {
-			logger.Error("Failed to parse server versions from workflow input")
-			return
-		}
-		logger.WithField("serverVersions", serverVersions).Info("Extracted server versions from workflow inputs")
-		var instanceType string
-		if strings.Contains(repoName, "desktop") {
-			instanceType = "desktop"
-		} else if strings.Contains(repoName, "mobile") {
-			instanceType = "mobile"
-		} else {
-			logger.Warn("Repository is neither desktop nor mobile, skipping CMT")
-			return
+	// CMT: a lightweight, scheduled trigger workflow (Config.CMTTriggerWorkflowName, e.g.
+	// "CMT Provisioner") fires in the desktop/mobile repo. matterwick provisions one instance
+	// per version in Config.CMTVersions() and dispatches compatibility-matrix-testing.yml.
+	// No inputs are read from the event — the version set is hardcoded in matterwick config —
+	// so this works despite GitHub's workflow_run payload not carrying workflow_dispatch inputs.
+	// Cleanup happens when the test workflow ("Compatibility Matrix Testing") completes, handled
+	// by the isE2ETestWorkflow branch below.
+	if s.Config.CMTTriggerWorkflowName != "" && workflowName == s.Config.CMTTriggerWorkflowName {
+		if payload.Action == "requested" {
+			logger.Info("CMT trigger workflow started, provisioning E2E servers for configured versions")
+			go s.handleCMTTrigger(owner, repoName, headBranch, headSHA, runID, logger)
 		}
-		go s.handleCMTWithServerVersions(owner, repoName, instanceType, headBranch, headSHA, serverVersions, runID, logger)
 		return
 	}
 
@@ -124,26 +103,13 @@ func (s *Server) handleWorkflowRunEventWithInputs(payload *WorkflowRunWebhookPay
 	}
 
 	// --- Test workflow completion: clean up provisioned instances ---
+	//
+	// Cleanup is keyed by head_sha against tracking-key suffixes ({repo}-...-{sha}).
+	// GitHub's workflow_run webhook payload does not include workflow_dispatch inputs,
+	// so SHA-based scanning is the canonical mechanism for matching provisioned instances
+	// (push, scheduled, and CMT tracking keys all end with "-{sha}").
 	if payload.Action == "completed" && s.isE2ETestWorkflow(workflowName) {
-		logger.Info("Test workflow completed, checking for instance cleanup")
-
-		// Primary: look up by mw_tracking_key embedded at dispatch time (immune to SHA races).
-		if trackingKey := payload.WorkflowRun.Inputs["mw_tracking_key"]; trackingKey != "" {
-			s.e2eInstancesLock.Lock()
-			instances := s.e2eInstances[trackingKey]
-			delete(s.e2eInstances, trackingKey)
-			s.e2eInstancesLock.Unlock()
-			if len(instances) > 0 {
-				logger.WithField("tracking_key", trackingKey).Info("Destroying instances by tracking key")
-				s.destroyE2EInstances(instances, logger)
-			} else {
-				logger.WithField("tracking_key", trackingKey).Debug("No in-memory instances for tracking key (matterwick restarted or already cleaned)")
-			}
-			return
-		}
-
-		// Fallback: SHA-based scan (runs dispatched before mw_tracking_key was introduced).
-		logger.Debug("No mw_tracking_key in workflow inputs, falling back to SHA-based instance cleanup")
+		logger.Info("Test workflow completed, cleaning up instances by SHA suffix match")
 		s.findAndDestroyInstancesBySHA(repoName, headSHA, logger)
 		return
 	}
@@ -216,10 +182,7 @@ func (s *Server) handleNightlyE2ETrigger(owner, repoName, branch, sha, triggerEv
 			s.destroyE2EInstances(instances, logger)
 			return
 		}
-		// Pass the tracking key so the workflow_run completed handler can clean up by
-		// direct key lookup rather than SHA suffix matching (immune to new commits during
-		// the ~30 min instance-creation window).
-		dispatchErr = s.dispatchDesktopE2EWorkflow(owner, repoName, branch, sha, instanceDetailsJSON, runType, key, nightly)
+		dispatchErr = s.dispatchDesktopE2EWorkflow(owner, repoName, branch, sha, instanceDetailsJSON, runType, nightly)
 	} else {
 		if len(instances) < 3 {
 			logger.Errorf("Expected 3 mobile instances, got %d", len(instances))
@@ -230,7 +193,7 @@ func (s *Server) handleNightlyE2ETrigger(owner, repoName, branch, sha, triggerEv
 			return
 		}
 		dispatchErr = s.dispatchMobileE2EWorkflow(owner, repoName, branch, sha,
-			instances[0].URL, instances[1].URL, instances[2].URL, "both", runType, key)
+			instances[0].URL, instances[1].URL, instances[2].URL, "both", runType)
 	}
 
 	if dispatchErr != nil {
@@ -298,6 +261,28 @@ func parseServerVersionsFromString(input string) []string {
 	return versions
 }
 
+// handleCMTTrigger is invoked when the scheduled CMT trigger workflow fires. It resolves the
+// instance type from the repo, reads the hardcoded server-version set from config, and hands
+// off to handleCMTWithServerVersions. The version list lives in matterwick (Config.CMTVersions),
+// so nothing needs to be read from the workflow_run event.
+func (s *Server) handleCMTTrigger(owner, repoName, branch, sha string, runID int64, logger logrus.FieldLogger) {
+	instanceType := "desktop"
+	if strings.Contains(repoName, "mobile") {
+		instanceType = "mobile"
+	} else if !strings.Contains(repoName, "desktop") {
+		logger.Warn("Repository is neither desktop nor mobile, skipping CMT trigger")
+		return
+	}
+
+	versions := s.Config.CMTVersions()
+	logger.WithFields(logrus.Fields{
+		"instanceType": instanceType,
+		"versions":     versions,
+	}).Info("Provisioning CMT instances for configured server versions")
+
+	s.handleCMTWithServerVersions(owner, repoName, instanceType, branch, sha, versions, runID, logger)
+}
+
 // handleCMTWithServerVersions orchestrates CMT testing: creates one instance per server
 // version, builds the CMT_MATRIX JSON, and dispatches compatibility-matrix-testing.yml once.
 func (s *Server) handleCMTWithServerVersions(repoOwner, repoName, instanceType, branch, sha string, serverVersions []string, runID int64, logger logrus.FieldLogger) {
@@ -373,9 +358,7 @@ func (s *Server) handleCMTWithServerVersions(repoOwner, repoName, instanceType,
 		return
 	}
 
-	// Pass the tracking key so the workflow_run completed handler can clean up by
-	// direct key lookup rather than SHA suffix matching.
-	if err := s.dispatchCMTWorkflow(repoOwner, repoName, sha, branch, cmtMatrixJSON, instanceType, key, runID, logger); err != nil {
+	if err := s.dispatchCMTWorkflow(repoOwner, repoName, sha, branch, cmtMatrixJSON, instanceType, runID, logger); err != nil {
 		logger.WithError(err).Error("Failed to dispatch compatibility-matrix-testing.yml")
 		s.e2eInstancesLock.Lock()
 		delete(s.e2eInstances, key)
@@ -488,19 +471,19 @@ func buildMobileCMTMatrixJSON(versions []string, instances []*E2EInstance) (stri
 }
 
 // dispatchCMTWorkflow dispatches compatibility-matrix-testing.yml with the populated
-// CMT_MATRIX JSON. trackingKey is the s.e2eInstances map key for this run; it is
-// embedded as "mw_tracking_key" in the workflow inputs so the workflow_run completed
-// handler can do a direct key lookup instead of fragile SHA suffix matching.
-// runID is the CMT provisioner workflow run ID, passed as cmt_run_id so the test workflow
-// can call back to Matterwick for instance cleanup.
-func (s *Server) dispatchCMTWorkflow(repoOwner, repoName, sha, branch, cmtMatrixJSON, instanceType, trackingKey string, runID int64, logger logrus.FieldLogger) error {
+// CMT_MATRIX JSON. runID is the CMT trigger workflow run ID, passed as cmt_run_id purely
+// for traceability/logging on the test workflow side.
+//
+// We intentionally do NOT pass a "mw_tracking_key" input: compatibility-matrix-testing.yml
+// does not declare it, and GitHub rejects a workflow_dispatch carrying an undeclared input
+// with a 422. Cleanup is driven by SHA-suffix matching when the test workflow completes.
+func (s *Server) dispatchCMTWorkflow(repoOwner, repoName, sha, branch, cmtMatrixJSON, instanceType string, runID int64, logger logrus.FieldLogger) error {
 	ctx := context.Background()
 	client := newGithubClient(s.Config.GithubAccessToken)
 
 	workflowInputs := map[string]interface{}{
-		"CMT_MATRIX":      cmtMatrixJSON,
-		"cmt_run_id":      fmt.Sprintf("%d", runID),
-		"mw_tracking_key": trackingKey,
+		"CMT_MATRIX": cmtMatrixJSON,
+		"cmt_run_id": fmt.Sprintf("%d", runID),
 	}
 	if instanceType == "desktop" {
 		workflowInputs["DESKTOP_VERSION"] = branch
@@ -616,18 +599,3 @@ func (s *Server) createCMTInstancesForVersion(repoName, instanceType, version, p
 	logger.WithField("instanceCount", len(instances)).Info("Instances created for version")
 	return instances, nil
 }
-
-// handleCMTRunCleanup is a best-effort fallback for CMT cleanup when the trigger workflow
-// completes. Because the CMT trigger is a lightweight workflow that completes in seconds —
-// well before the 30-minute provisioning goroutine stores instances — this function will
-// most often find nothing. The primary cleanup path is findAndDestroyInstancesBySHA,
-// triggered when compatibility-matrix-testing.yml completes.
-func (s *Server) handleCMTRunCleanup(repoName, sha string, logger logrus.FieldLogger) {
-	logger = logger.WithFields(logrus.Fields{
-		"repo": repoName,
-		"sha":  sha,
-		"type": "cmt_cleanup_fallback",
-	})
-	logger.Debug("CMT trigger completed — sha-based cleanup is the primary path")
-	s.findAndDestroyInstancesBySHA(repoName, sha, logger)
-}