Skip to content

fix(runner): disable warm starts after SIGTERM #2316

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 4 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions .changeset/cyan-news-design.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"trigger.dev": patch
---

Allow any runs to finish after SIGTERM but disable warm starts
22 changes: 20 additions & 2 deletions packages/cli-v3/src/entryPoints/managed/controller.ts
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,9 @@ export class ManagedRunController {
private readonly logger: RunLogger;
private readonly taskRunProcessProvider: TaskRunProcessProvider;

private warmStartEnabled = true;
private warmStartCount = 0;

private restoreCount = 0;

private notificationCount = 0;
Expand Down Expand Up @@ -103,7 +105,12 @@ export class ManagedRunController {
runId: this.runFriendlyId,
message: "Received SIGTERM, stopping worker",
});
await this.stop();

// Disable warm starts
this.warmStartEnabled = false;

// ..now we wait for any active runs to finish
// SIGKILL will handle the rest, nothing to do here
});
}

Expand Down Expand Up @@ -276,6 +283,14 @@ export class ManagedRunController {
* the process on any errors or when no runs are available after the configured duration.
*/
private async waitForNextRun() {
if (!this.warmStartEnabled) {
this.sendDebugLog({
runId: this.runFriendlyId,
message: "waitForNextRun: warm starts disabled, shutting down",
});
this.exitProcess(this.successExitCode);
}

this.sendDebugLog({
runId: this.runFriendlyId,
message: "waitForNextRun()",
Expand Down Expand Up @@ -548,7 +563,7 @@ export class ManagedRunController {
return;
}

async stop() {
async cancelRunsAndExitProcess() {
this.sendDebugLog({
runId: this.runFriendlyId,
message: "Shutting down",
Expand Down Expand Up @@ -578,6 +593,9 @@ export class ManagedRunController {

// Close the socket
this.socket.close();

// Exit the process
this.exitProcess(this.successExitCode);
}

sendDebugLog(opts: SendDebugLogOptions) {
Expand Down