exit code cleanup (#753)

- use consistent enums for exit codes
- add disk space check on startup and add OutOfSpace exit code (3)
- preparation for #584

src/crawler.ts

@@ -27,6 +27,7 @@ import {
   interpolateFilename,
   checkDiskUtilization,
   S3StorageSync,
+  isDiskFull,
 } from "./util/storage.js";
 import { ScreenCaster, WSTransport } from "./util/screencaster.js";
 import { Screenshots } from "./util/screenshots.js";
@@ -46,6 +47,7 @@ import {
   ExtractSelector,
   PAGE_OP_TIMEOUT_SECS,
   SITEMAP_INITIAL_FETCH_TIMEOUT_SECS,
+  ExitCodes,
 } from "./util/constants.js";
 
 import { AdBlockRules, BlockRuleDecl, BlockRules } from "./util/blockrules.js";
@@ -462,6 +464,15 @@ export class Crawler {
   }
 
   async bootstrap() {
+    if (await isDiskFull(this.params.cwd)) {
+      logger.fatal(
+        "Out of disk space, exiting",
+        {},
+        "general",
+        ExitCodes.OutOfSpace,
+      );
+    }
+
     const subprocesses: ChildProcess[] = [];
 
     const redisUrl = this.params.redisStoreUrl || "redis://localhost:6379/0";
@@ -572,7 +583,7 @@ export class Crawler {
     await this.bootstrap();
 
     let status = "done";
-    let exitCode = 0;
+    let exitCode = ExitCodes.Success;
 
     try {
       await this.crawl();
@@ -587,12 +598,14 @@ export class Crawler {
           logger.info("Crawl gracefully stopped on request");
         } else if (this.interrupted) {
           status = "interrupted";
-          exitCode = this.browserCrashed ? 10 : 11;
+          exitCode = this.browserCrashed
+            ? ExitCodes.BrowserCrashed
+            : ExitCodes.InterruptedGraceful;
         }
       }
     } catch (e) {
       logger.error("Crawl failed", e);
-      exitCode = 9;
+      exitCode = ExitCodes.Failed;
       status = "failing";
       if (await this.crawlState.incFailCount()) {
         status = "failed";
@@ -1419,11 +1432,11 @@ self.__bx_behaviors.selectMainBehavior();
 
   async checkCanceled() {
     if (this.crawlState && (await this.crawlState.isCrawlCanceled())) {
-      await this.setStatusAndExit(0, "canceled");
+      await this.setStatusAndExit(ExitCodes.Success, "canceled");
     }
   }
 
-  async setStatusAndExit(exitCode: number, status: string) {
+  async setStatusAndExit(exitCode: ExitCodes, status: string) {
     logger.info(`Exiting, Crawl status: ${status}`);
 
     await this.closeLog();
@@ -1442,11 +1455,14 @@ self.__bx_behaviors.selectMainBehavior();
       await closeWorkers(0);
       await this.closeFiles();
       if (!this.done) {
-        await this.setStatusAndExit(13, "interrupted");
+        await this.setStatusAndExit(
+          ExitCodes.InterruptedImmediate,
+          "interrupted",
+        );
         return;
       }
     }
-    await this.setStatusAndExit(0, "done");
+    await this.setStatusAndExit(ExitCodes.Success, "done");
   }
 
   async isCrawlRunning() {
@@ -1455,7 +1471,7 @@ self.__bx_behaviors.selectMainBehavior();
     }
 
     if (await this.crawlState.isCrawlCanceled()) {
-      await this.setStatusAndExit(0, "canceled");
+      await this.setStatusAndExit(ExitCodes.Success, "canceled");
       return false;
     }
 

src/create-login-profile.ts

@@ -15,7 +15,7 @@ import { Browser } from "./util/browser.js";
 import { initStorage } from "./util/storage.js";
 import { CDPSession, Page, PuppeteerLifeCycleEvent } from "puppeteer-core";
 import { getInfoString } from "./util/file_reader.js";
-import { DISPLAY } from "./util/constants.js";
+import { DISPLAY, ExitCodes } from "./util/constants.js";
 import { initProxy } from "./util/proxy.js";
 //import { sleep } from "./util/timing.js";
 
@@ -147,7 +147,7 @@ function getDefaultWindowSize() {
 
 function handleTerminate(signame: string) {
   logger.info(`Got signal ${signame}, exiting`);
-  process.exit(1);
+  process.exit(ExitCodes.GenericError);
 }
 
 async function main() {
@@ -478,7 +478,10 @@ class InteractiveBrowser {
     this.shutdownWait = params.shutdownWait * 1000;
 
     if (this.shutdownWait) {
-      this.shutdownTimer = setTimeout(() => process.exit(0), this.shutdownWait);
+      this.shutdownTimer = setTimeout(
+        () => process.exit(ExitCodes.Success),
+        this.shutdownWait,
+      );
       logger.debug(
         `Shutting down in ${this.shutdownWait}ms if no ping received`,
       );
@@ -602,7 +605,7 @@ class InteractiveBrowser {
           // eslint-disable-next-line @typescript-eslint/no-explicit-any
           clearTimeout(this.shutdownTimer as any);
           this.shutdownTimer = setTimeout(
-            () => process.exit(0),
+            () => process.exit(ExitCodes.Success),
             this.shutdownWait,
           );
           logger.debug(
@@ -682,7 +685,7 @@ class InteractiveBrowser {
           logger.warn("HTTP Error", e);
         }
 
-        setTimeout(() => process.exit(0), 200);
+        setTimeout(() => process.exit(ExitCodes.Success), 200);
         return;
 
       case "/createProfile":
@@ -708,7 +711,7 @@ class InteractiveBrowser {
           logger.warn("HTTP Error", e);
         }
 
-        setTimeout(() => process.exit(0), 200);
+        setTimeout(() => process.exit(ExitCodes.Success), 200);
         return;
     }
 

src/main.ts

@@ -5,6 +5,7 @@ import { setExitOnRedisError } from "./util/redis.js";
 import { Crawler } from "./crawler.js";
 import { ReplayCrawler } from "./replaycrawler.js";
 import fs from "node:fs";
+import { ExitCodes } from "./util/constants.js";
 
 let crawler: Crawler | null = null;
 
@@ -15,12 +16,12 @@ async function handleTerminate(signame: string) {
   logger.info(`${signame} received...`);
   if (!crawler || !crawler.crawlState) {
     logger.error("error: no crawler running, exiting");
-    process.exit(1);
+    process.exit(ExitCodes.GenericError);
   }
 
   if (crawler.done) {
     logger.info("success: crawler done, exiting");
-    process.exit(0);
+    process.exit(ExitCodes.Success);
   }
 
   setExitOnRedisError();

src/util/constants.ts

@@ -56,3 +56,15 @@ export const BEHAVIOR_TYPES = [
 ];
 
 export const DISPLAY = ":99";
+
+export enum ExitCodes {
+  Success = 0,
+  GenericError = 1,
+  Failed = 9,
+  OutOfSpace = 3,
+  BrowserCrashed = 10,
+  InterruptedGraceful = 11,
+  InterruptedImmediate = 13,
+  Fatal = 17,
+  ProxyError = 21,
+}

src/util/logger.ts

@@ -3,6 +3,7 @@
 
 import { Writable } from "node:stream";
 import { RedisCrawlState } from "./state.js";
+import { ExitCodes } from "./constants.js";
 
 // RegExp.prototype.toJSON = RegExp.prototype.toString;
 Object.defineProperty(RegExp.prototype, "toJSON", {
@@ -73,7 +74,7 @@ class Logger {
   contexts: LogContext[] = [];
   excludeContexts: LogContext[] = [];
   crawlState?: RedisCrawlState | null = null;
-  fatalExitCode = 17;
+  fatalExitCode: ExitCodes = ExitCodes.Fatal;
 
   setDefaultFatalExitCode(exitCode: number) {
     this.fatalExitCode = exitCode;
@@ -143,8 +144,12 @@ class Logger {
     };
     const string = JSON.stringify(dataToLog);
     console.log(string);
-    if (this.logStream) {
-      this.logStream.write(string + "\n");
+    try {
+      if (this.logStream) {
+        this.logStream.write(string + "\n");
+      }
+    } catch (e) {
+      //
     }
 
     const toLogToRedis = ["error", "fatal"];
@@ -179,7 +184,7 @@ class Logger {
     message: string,
     data = {},
     context: LogContext = "general",
-    exitCode = 0,
+    exitCode = ExitCodes.Success,
   ) {
     exitCode = exitCode || this.fatalExitCode;
     this.logAsJSON(`${message}. Quitting`, data, context, "fatal");

src/util/proxy.ts

@@ -7,7 +7,7 @@ import { logger } from "./logger.js";
 
 import { socksDispatcher } from "fetch-socks";
 import type { SocksProxyType } from "socks/typings/common/constants.js";
-import { FETCH_HEADERS_TIMEOUT_SECS } from "./constants.js";
+import { ExitCodes, FETCH_HEADERS_TIMEOUT_SECS } from "./constants.js";
 
 const SSH_PROXY_LOCAL_PORT = 9722;
 
@@ -194,7 +194,7 @@ export async function runSSHD(params: Record<string, any>, detached: boolean) {
         code: proc.exitCode,
       },
       "proxy",
-      21,
+      ExitCodes.ProxyError,
     );
     return;
   }

src/util/storage.ts

@@ -231,6 +231,12 @@ export async function getDirSize(dir: string): Promise<number> {
   return size;
 }
 
+export async function isDiskFull(collDir: string) {
+  const diskUsage: Record<string, string> = await getDiskUsage(collDir);
+  const usedPercentage = parseInt(diskUsage["Use%"].slice(0, -1));
+  return usedPercentage >= 99;
+}
+
 export async function checkDiskUtilization(
   collDir: string,
   // TODO: Fix this the next time the file is edited.