feat(machined): add panic/force mode reboot

In certain situations, Talos's shutdown/reboot sequence hangs while
waiting for services/mounts to be gracefully stopped (see:
#11775).

This patch adds a forceful mode to the reboot sequence (`talosctl reboot
--mode force`) that bypasses graceful userspace teardown and hard
reboots the machine.

Signed-off-by: Laura Brehm <[email protected]>

Author: laurazard

api/machine/machine.proto

@@ -143,6 +143,7 @@ message RebootRequest {
   enum Mode {
     DEFAULT = 0;
     POWERCYCLE = 1;
+    FORCE = 2;
   }
   Mode mode = 1;
 }

cmd/talosctl/cmd/talos/reboot.go

@@ -39,6 +39,8 @@ var rebootCmd = &cobra.Command{
 		// skips kexec and reboots with power cycle
 		case "powercycle":
 			opts = append(opts, client.WithPowerCycle)
+		case "force":
+			opts = append(opts, client.WithForce)
 		case "default":
 		default:
 			return fmt.Errorf("invalid reboot mode: %q", rebootCmdFlags.mode)

cmd/talosctl/pkg/talos/action/node.go

@@ -286,6 +286,12 @@ func (a *nodeTracker) handleEvent(event client.Event) error {
 			Status:  reporter.StatusRunning,
 		})
 
+		// TODO: if we first do stopAllServices, that's already handled
+		// and we don't need special handling for the reboot "error" case.
+		if msg.GetSequence() == "reboot" {
+			return retry.ExpectedErrorf("reboot sequence completed")
+		}
+
 		if errStr != "" {
 			return fmt.Errorf("sequence error: %s", msg.GetError().GetMessage())
 		}

internal/app/machined/pkg/runtime/sequencer.go

@@ -147,7 +147,7 @@ type Sequencer interface {
 	Boot(Runtime) []Phase
 	Initialize(Runtime) []Phase
 	Install(Runtime) []Phase
-	Reboot(Runtime) []Phase
+	Reboot(Runtime, *machine.RebootRequest) []Phase
 	Reset(Runtime, ResetOptions) []Phase
 	Shutdown(Runtime, *machine.ShutdownRequest) []Phase
 	StageUpgrade(Runtime, *machine.UpgradeRequest) []Phase

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller.go

@@ -381,7 +381,19 @@ func (c *Controller) phases(seq runtime.Sequence, data any) ([]runtime.Phase, er
 
 		phases = c.s.Shutdown(c.r, in)
 	case runtime.SequenceReboot:
-		phases = c.s.Reboot(c.r)
+		// TODO: we're both using *machine.XxxRequest and
+		// runtime.XxxOptions types here. Is one of these preferred?
+		// Might be kind of confusing to be passing a request here
+		// when the request also gets passed to the specific tasks
+		// via controller.runTask().
+		var in *machine.RebootRequest
+		if req, ok := data.(*machine.RebootRequest); ok {
+			// TODO: complain if missing? are there older clients
+			// that might not send a `machine.RebootRequest`?
+			in = req
+		}
+
+		phases = c.s.Reboot(c.r, in)
 	case runtime.SequenceUpgrade:
 		in, ok := data.(*machine.UpgradeRequest)
 		if !ok {

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_controller_test.go

@@ -44,7 +44,7 @@ func (m *mockSequencer) Install(r runtime.Runtime) []runtime.Phase {
 	return m.phases[runtime.SequenceInstall]
 }
 
-func (m *mockSequencer) Reboot(r runtime.Runtime) []runtime.Phase {
+func (m *mockSequencer) Reboot(r runtime.Runtime, req *machine.RebootRequest) []runtime.Phase {
 	return m.phases[runtime.SequenceReboot]
 }
 

internal/app/machined/pkg/runtime/v1alpha1/v1alpha1_sequencer.go

@@ -233,18 +233,22 @@ func (*Sequencer) Boot(r runtime.Runtime) []runtime.Phase {
 }
 
 // Reboot is the reboot sequence.
-func (*Sequencer) Reboot(r runtime.Runtime) []runtime.Phase {
-	phases := PhaseList{}.Append(
-		"cleanup",
-		StopAllPods,
-	).Append(
-		"dbus",
-		StopDBus,
-	).
-		AppendList(stopAllPhaselist(r, true)).
-		Append("reboot", Reboot)
+func (*Sequencer) Reboot(r runtime.Runtime, in *machineapi.RebootRequest) []runtime.Phase {
+	if in.GetMode() == machineapi.RebootRequest_FORCE {
+		return PhaseList{}.
+			Append("reboot", Reboot)
+	}
 
-	return phases
+	return PhaseList{}.
+		Append(
+			"cleanup",
+			StopAllPods,
+		).
+		Append(
+			"dbus",
+			StopDBus,
+		).
+		AppendList(stopAllPhaselist(r, true))
 }
 
 // Reset is the reset sequence.

internal/integration/api/reboot.go

@@ -10,12 +10,14 @@ import (
 	"context"
 	"fmt"
 	"sync"
+	"sync/atomic"
 	"testing"
 	"time"
 
 	"github.com/siderolabs/go-retry/retry"
 
 	"github.com/siderolabs/talos/internal/integration/base"
+	machineapi "github.com/siderolabs/talos/pkg/machinery/api/machine"
 	"github.com/siderolabs/talos/pkg/machinery/client"
 	"github.com/siderolabs/talos/pkg/machinery/config/machine"
 )
@@ -71,6 +73,71 @@ func (suite *RebootSuite) TestRebootNodeByNode() {
 	}
 }
 
+// TestForcedReboot force-reboots cluster node by node,
+// ensuring that the 'cleanup' phase/'stopAllPods' task doesn't run.
+func (suite *RebootSuite) TestForcedReboot() { //nolint:gocyclo
+	if !suite.Capabilities().SupportsReboot {
+		suite.T().Skip("cluster doesn't support reboots")
+	}
+
+	nodes := suite.DiscoverNodeInternalIPs(suite.ctx)
+	suite.Require().NotEmpty(nodes)
+
+	for _, node := range nodes {
+		suite.T().Log("force rebooting node", node)
+
+		nodeCtx := client.WithNodes(suite.ctx, node)
+
+		var (
+			sawStopAllPods  atomic.Bool
+			sawCleanupPhase atomic.Bool
+		)
+
+		// watch events so we can verify graceful teardown did not happen
+		watchCtx, watchCancel := context.WithCancel(nodeCtx)
+		eventsCh := make(chan client.EventResult)
+		suite.Require().NoError(suite.Client.EventsWatchV2(watchCtx, eventsCh))
+
+		go func() {
+			for {
+				select {
+				case <-watchCtx.Done():
+					return
+				case ev := <-eventsCh:
+					if ev.Error != nil {
+						continue
+					}
+
+					switch msg := ev.Event.Payload.(type) {
+					case *machineapi.TaskEvent:
+						if msg.GetTask() == "stopAllPods" {
+							sawStopAllPods.Store(true)
+						}
+					case *machineapi.PhaseEvent:
+						if msg.GetPhase() == "cleanup" {
+							sawCleanupPhase.Store(true)
+						}
+					}
+				}
+			}
+		}()
+
+		suite.AssertRebooted(
+			suite.ctx, node, func(nodeCtx context.Context) error {
+				return base.IgnoreGRPCUnavailable(suite.Client.Reboot(nodeCtx, client.WithForce))
+			}, 10*time.Minute,
+			suite.CleanupFailedPods,
+		)
+
+		watchCancel()
+
+		suite.Require().Falsef(sawCleanupPhase.Load(), "cleanup phase must not run during forced reboot")
+		suite.Require().Falsef(sawStopAllPods.Load(), "stopAllPods task must not run during forced reboot")
+	}
+
+	suite.WaitForBootDone(suite.ctx)
+}
+
 // TestRebootMultiple reboots a node, issues consequent reboots
 // reboot should cancel boot sequence, and cancel another reboot.
 func (suite *RebootSuite) TestRebootMultiple() {

pkg/machinery/api/machine/machine.pb.go

@@ -337,6 +337,11 @@ func WithPowerCycle(req *machineapi.RebootRequest) {
 	req.Mode = machineapi.RebootRequest_POWERCYCLE
 }
 
+// WithForce option runs the Reboot fun in force mode.
+func WithForce(req *machineapi.RebootRequest) {
+	req.Mode = machineapi.RebootRequest_FORCE
+}
+
 // Reboot implements the proto.MachineServiceClient interface.
 func (c *Client) Reboot(ctx context.Context, opts ...RebootMode) error {
 	_, err := c.RebootWithResponse(ctx, opts...)