Fix queued snapshot status handler (#1963)

* handle queued status change gracefully

* add resource monitor to references

* add resource monitor example

* improve runtime manager debug logs

* fix resource monitor example

* add changeset

Author: nicktrn

.changeset/wet-deers-think.md

@@ -0,0 +1,6 @@
+---
+"trigger.dev": patch
+"@trigger.dev/core": patch
+---
+
+Fix QUEUED status snapshot handler

packages/cli-v3/src/entryPoints/managed/execution.ts

@@ -269,6 +269,13 @@ export class RunExecution {
         this.abortExecution();
         return;
       }
+      case "QUEUED": {
+        this.sendDebugLog("Run was re-queued", snapshotMetadata);
+
+        // Pretend we've just suspended the run. This will kill the process without failing the run.
+        await this.taskRunProcess?.suspend();
+        return;
+      }
       case "FINISHED": {
         this.sendDebugLog("Run is finished", snapshotMetadata);
 
@@ -402,8 +409,7 @@ export class RunExecution {
 
         return;
       }
-      case "RUN_CREATED":
-      case "QUEUED": {
+      case "RUN_CREATED": {
         this.sendDebugLog("Invalid status change", snapshotMetadata);
 
         this.abortExecution();

packages/core/src/v3/runtime/managedRuntimeManager.ts

@@ -27,12 +27,10 @@ export class ManagedRuntimeManager implements RuntimeManager {
     private ipc: ExecutorToWorkerProcessConnection,
     private showLogs: boolean
   ) {
-    setTimeout(() => {
-      this.log("Runtime status", {
-        resolversbyWaitId: this.resolversByWaitId.keys(),
-        resolversByWaitpoint: this.resolversByWaitpoint.keys(),
-      });
-    }, 1000);
+    // Log out the runtime status on a long interval to help debug stuck executions
+    setInterval(() => {
+      this.log("[DEBUG] ManagedRuntimeManager status", this.status);
+    }, 300_000);
   }
 
   disable(): void {
@@ -178,14 +176,14 @@ export class ManagedRuntimeManager implements RuntimeManager {
     }
 
     if (!waitId) {
-      this.log("No waitId found for waitpoint", waitpoint);
+      this.log("No waitId found for waitpoint", { ...this.status, ...waitpoint });
       return;
     }
 
     const resolve = this.resolversByWaitId.get(waitId);
 
     if (!resolve) {
-      this.log("No resolver found for waitId", waitId);
+      this.log("No resolver found for waitId", { ...this.status, waitId });
       return;
     }
 
@@ -227,4 +225,11 @@ export class ManagedRuntimeManager implements RuntimeManager {
     if (!this.showLogs) return;
     console.log(`[${new Date().toISOString()}] ${message}`, args);
   }
+
+  private get status() {
+    return {
+      resolversbyWaitId: Array.from(this.resolversByWaitId.keys()),
+      resolversByWaitpoint: Array.from(this.resolversByWaitpoint.keys()),
+    };
+  }
 }