feat: support output directory for attachments

__tests__/reporters/__snapshots__/base.test.ts.snap

@@ -3,10 +3,10 @@
 exports[`base reporter render hook errors without steps 1`] = `
 "
 Journey: j1
-   
+
    Error: before hook failed
 
-No tests found! (0 ms) 
+No tests found! (0 ms)
 
 "
 `;
@@ -15,11 +15,11 @@ exports[`base reporter writes each step to the FD 1`] = `
 "
 Journey: j1
    ✖  Step: 's1' failed (1000 ms)
-   
+
    Error: step failed
 
 
- 1 failed (0 ms) 
+ 1 failed (0 ms)
 
 "
 `;

__tests__/reporters/base.test.ts

@@ -70,7 +70,14 @@ describe('base reporter', () => {
     reporter.onJourneyStart(j1, { timestamp });
     reporter.onStepEnd(j1, s1, {});
     reporter.onEnd();
-    expect((await readAndCloseStream()).toString()).toMatchSnapshot();
+    const output = await readAndCloseStream();
+    expect(output.toString().trim()).toContain(`Journey: j1
+   ✖  Step: 's1' failed (1000 ms)
+   ---
+      stack: |-
+        Error: step failed
+   ---`);
+    expect(output.toString().trim()).toContain(`.screenshots/j1/s1.jpg`);
   });
 
   it('render hook errors without steps', async () => {

__tests__/reporters/json.test.ts

@@ -30,14 +30,14 @@ import SonicBoom from 'sonic-boom';
 import { journey } from '../../src/core';
 import JSONReporter, {
   formatNetworkFields,
-  gatherScreenshots,
   getScreenshotBlocks,
   redactKeys,
 } from '../../src/reporters/json';
 import * as helpers from '../../src/helpers';
 import { NETWORK_INFO } from '../fixtures/networkinfo';
 import { StatusValue } from '../../src/common_types';
 import { tJourney, tStep } from '../utils/test-config';
+import { gatherScreenshots } from '../../src/reporters/utils';
 
 /**
  * Mock package version to avoid breaking JSON payload

src/cli.ts

@@ -136,6 +136,10 @@ program
     'Turns off default network throttling.',
     parseThrottling
   )
+  .option(
+    '--outputDir <outputDir>',
+    'output directory for screenshots and videos'
+  )
   .addOption(playwrightOpts)
   .version(version)
   .description('Run synthetic tests')

src/common_types.ts

@@ -192,6 +192,7 @@ export type PluginOutput = {
   browserconsole?: Array<BrowserMessage>;
   traces?: Array<TraceOutput>;
   metrics?: PerfMetrics;
+  attachments?: Array<Attachment>;
 };
 
 export type ScreenshotOptions = 'on' | 'off' | 'only-on-failure';
@@ -215,6 +216,7 @@ type BaseArgs = {
   config?: string;
   auth?: string;
   outfd?: number;
+  outputDir?: string;
   wsEndpoint?: string;
   pauseOnError?: boolean;
   playwrightOptions?: PlaywrightOptions;
@@ -285,11 +287,35 @@ export type SyntheticsConfig = {
   project?: ProjectSettings;
 };
 
+export type Attachment = {
+  /**
+   * Attachment name.
+   */
+  name: string;
+
+  /**
+   * Content type of this attachment to properly present in the report, for example `'application/json'` or
+   * `'image/png'`.
+   */
+  contentType: string;
+
+  /**
+   * Optional path on the filesystem to the attached file.
+   */
+  path?: string;
+
+  /**
+   * Optional attachment body used instead of a file.
+   */
+  body?: Buffer;
+};
+
 /** Runner Payload types */
 export type JourneyResult = Partial<Journey> & {
   networkinfo?: PluginOutput['networkinfo'];
   browserconsole?: PluginOutput['browserconsole'];
   stepsresults?: Array<StepResult>;
+  attachments?: Array<Attachment>;
 };
 
 export type TestError = {

src/core/runner.ts

@@ -23,29 +23,26 @@
  *
  */
 
-import { join } from 'path';
-import { mkdir, rm, writeFile } from 'fs/promises';
+import { mkdir, rm } from 'fs/promises';
 import { Journey } from '../dsl/journey';
 import { Step } from '../dsl/step';
-import { reporters, Reporter } from '../reporters';
+import { Reporter, reporters } from '../reporters';
 import {
   CACHE_PATH,
-  monotonicTimeInSeconds,
   getTimestamp,
+  monotonicTimeInSeconds,
   runParallel,
-  generateUniqueId,
 } from '../helpers';
 import {
-  HooksCallback,
-  HooksArgs,
   Driver,
-  Screenshot,
-  RunOptions,
+  HooksArgs,
+  HooksCallback,
   JourneyResult,
-  StepResult,
   PushOptions,
+  RunOptions,
+  StepResult,
 } from '../common_types';
-import { PerformanceManager, filterBrowserMessages } from '../plugins';
+import { filterBrowserMessages } from '../plugins';
 import { Gatherer } from './gatherer';
 import { log } from './logger';
 import { Monitor, MonitorConfig } from '../dsl/monitor';
@@ -65,7 +62,7 @@ export interface RunnerInfo {
    */
   readonly currentJourney: Journey | undefined;
   /**
-   * All registerd journeys
+   * All registered journeys
    */
   readonly journeys: Journey[];
 }
@@ -76,7 +73,6 @@ export default class Runner implements RunnerInfo {
   #currentJourney?: Journey = null;
   #journeys: Journey[] = [];
   #hooks: SuiteHooks = { beforeAll: [], afterAll: [] };
-  #screenshotPath = join(CACHE_PATH, 'screenshots');
   #driver?: Driver;
   #browserDelay = -1;
   #hookError: Error | undefined;
@@ -95,35 +91,6 @@ export default class Runner implements RunnerInfo {
     return this.#hooks;
   }
 
-  private async captureScreenshot(page: Driver['page'], step: Step) {
-    try {
-      const buffer = await page.screenshot({
-        type: 'jpeg',
-        quality: 80,
-        timeout: 5000,
-      });
-      /**
-       * Write the screenshot image buffer with additional details (step
-       * information) which could be extracted at the end of
-       * each journey without impacting the step timing information
-       */
-      const fileName = `${generateUniqueId()}.json`;
-      const screenshot: Screenshot = {
-        step,
-        timestamp: getTimestamp(),
-        data: buffer.toString('base64'),
-      };
-      await writeFile(
-        join(this.#screenshotPath, fileName),
-        JSON.stringify(screenshot)
-      );
-      log(`Runner: captured screenshot for (${step.name})`);
-    } catch (_) {
-      // Screenshot may fail sometimes, log and continue.
-      log(`Runner: failed to capture screenshot for (${step.name})`);
-    }
-  }
-
   _addHook(type: HookType, callback: HooksCallback) {
     this.#hooks[type].push(callback);
   }
@@ -178,12 +145,12 @@ export default class Runner implements RunnerInfo {
      * Set up the corresponding reporter and fallback
      * to default reporter if not provided
      */
-    const { reporter, outfd, dryRun } = options;
+    const { reporter, outfd, dryRun, outputDir } = options;
     const Reporter =
       typeof reporter === 'function'
         ? reporter
         : reporters[reporter] || reporters['default'];
-    this.#reporter = new Reporter({ fd: outfd, dryRun });
+    this.#reporter = new Reporter({ fd: outfd, dryRun, outputDir });
   }
 
   async #runBeforeAllHook(args: HooksArgs) {
@@ -208,7 +175,6 @@ export default class Runner implements RunnerInfo {
 
   async #runStep(step: Step, options: RunOptions): Promise<StepResult> {
     log(`Runner: start step (${step.name})`);
-    const { metrics, screenshots, filmstrips, trace } = options;
     /**
      * URL needs to be the first navigation request of any step
      * Listening for request solves the case where `about:blank` would be
@@ -223,14 +189,12 @@ export default class Runner implements RunnerInfo {
     this.#driver.context.on('request', captureUrl);
 
     const data: StepResult = {};
-    const traceEnabled = trace || filmstrips;
     try {
       /**
        * Set up plugin manager context and also register
        * step level plugins
        */
-      Gatherer.pluginManager.onStep(step);
-      traceEnabled && (await Gatherer.pluginManager.start('trace'));
+      await Gatherer.pluginManager.onStep(step, options);
       // invoke the step callback by extracting to a variable to get better stack trace
       const cb = step.cb;
       await cb();
@@ -239,32 +203,8 @@ export default class Runner implements RunnerInfo {
       step.status = 'failed';
       step.error = error;
     } finally {
-      /**
-       * Collect all step level metrics and trace events
-       */
-      if (metrics) {
-        data.pagemetrics = await (
-          Gatherer.pluginManager.get('performance') as PerformanceManager
-        ).getMetrics();
-      }
-      if (traceEnabled) {
-        const traceOutput = await Gatherer.pluginManager.stop('trace');
-        Object.assign(data, traceOutput);
-      }
-      /**
-       * Capture screenshot for the newly created pages
-       * via popup or new windows/tabs
-       *
-       * Last open page will get us the correct screenshot
-       */
-      const pages = this.#driver.context.pages();
-      const page = pages[pages.length - 1];
-      if (page) {
-        step.url ??= page.url();
-        if (screenshots && screenshots !== 'off') {
-          await this.captureScreenshot(page, step);
-        }
-      }
+      // Run all the registered plugins for the current step
+      await Gatherer.pluginManager.onStepEnd(step, options, data);
     }
     log(`Runner: end step (${step.name})`);
     return data;
@@ -311,18 +251,15 @@ export default class Runner implements RunnerInfo {
     journey._startTime = monotonicTimeInSeconds();
     this.#driver = await Gatherer.setupDriver(options);
     await Gatherer.beginRecording(this.#driver, options);
-    /**
-     * For each journey we create the screenshots folder for
-     * caching all screenshots and clear them at end of each journey
-     */
-    await mkdir(this.#screenshotPath, { recursive: true });
+    await Gatherer.pluginManager.onJourneyStart();
+
     const params = options.params;
     this.#reporter?.onJourneyStart?.(journey, {
       timestamp: getTimestamp(),
       params,
     });
     /**
-     * Exeucute the journey callback which registers the steps for current journey
+     * Execute the journey callback which registers the steps for current journey
      */
     journey.cb({ ...this.#driver, params, info: this });
   }
@@ -344,11 +281,12 @@ export default class Runner implements RunnerInfo {
       options,
       networkinfo: pOutput.networkinfo,
       browserconsole: bConsole,
+      attachments: pOutput.attachments,
     });
     await Gatherer.endRecording();
     await Gatherer.dispose(this.#driver);
-    // clear screenshots cache after each journey
-    await rm(this.#screenshotPath, { recursive: true, force: true });
+    await Gatherer.pluginManager.onJourneyEnd();
+
     return Object.assign(result, {
       networkinfo: pOutput.networkinfo,
       browserconsole: bConsole,
@@ -509,9 +447,9 @@ export default class Runner implements RunnerInfo {
     const { dryRun, grepOpts } = options;
 
     // collect all journeys with `.only` annotation and skip the rest
-    const onlyJournerys = this.#journeys.filter(j => j.only);
-    if (onlyJournerys.length > 0) {
-      this.#journeys = onlyJournerys;
+    const onlyJourneys = this.#journeys.filter(j => j.only);
+    if (onlyJourneys.length > 0) {
+      this.#journeys = onlyJourneys;
     } else {
       // filter journeys based on tags and skip annotations
       this.#journeys = this.#journeys.filter(
@@ -549,10 +487,9 @@ export default class Runner implements RunnerInfo {
     this.#browserDelay = monotonicTimeInSeconds() - browserStart;
 
     for (const journey of this.#journeys) {
-      const journeyResult: JourneyResult = this.#hookError
+      result[journey.name] = this.#hookError
         ? await this.#runFakeJourney(journey, options)
         : await this._runJourney(journey, options);
-      result[journey.name] = journeyResult;
     }
     await Gatherer.stop();
     return result;