feat(ecau): add ability to export debug logs and HTTP responses for bug reports

build/rollup.ts

@@ -15,6 +15,8 @@ import postcss from 'rollup-plugin-postcss';
 import progress from 'rollup-plugin-progress';
 import { minify } from 'terser';
 
+import { deduplicateArray } from '@lib/util/array';
+
 import { parseChangelogEntries } from './changelog';
 import { consts } from './plugin-consts';
 import { logger } from './plugin-logger';
@@ -249,7 +251,7 @@ function getVendorMinifiedPreamble(chunk: Readonly<RenderedChunk>): string {
             .slice(0, module.startsWith('@') ? 2 : 1)
             .join('/'));
 
-    const uniqueBundledModules = [...new Set(bundledModules)];
+    const uniqueBundledModules = deduplicateArray(bundledModules);
     if ('\u0000rollupPluginBabelHelpers.js' in chunk.modules) {
         uniqueBundledModules.unshift('babel helpers');
     }

src/lib/MB/URLs.ts

@@ -1,3 +1,4 @@
+import { deduplicateArray } from '@lib/util/array';
 import { request } from '@lib/util/request';
 
 import type { ReleaseAdvRel, URLAdvRel } from './advanced-relationships';
@@ -21,7 +22,7 @@ export async function getURLsForRelease(releaseId: string, options?: { excludeEn
     }
     let urls = urlARs.map((ar) => ar.url.resource);
     if (excludeDuplicates) {
-        urls = [...new Set(urls)];
+        urls = deduplicateArray(urls);
     }
 
     return urls.flatMap((url) => {

src/lib/logging/collectorSink.ts

@@ -0,0 +1,46 @@
+import type { LoggingSink } from './sink';
+import { LogLevel } from './levels';
+
+interface LogRecord {
+    level: string;
+    message: string;
+    timestamp: number;
+    exception?: unknown;
+}
+
+export class CollectorSink implements LoggingSink {
+    private readonly records: LogRecord[];
+
+    public constructor() {
+        this.records = [];
+    }
+
+    private saveMessage(level: string, message: string, exception?: unknown): void {
+        this.records.push({
+            level,
+            message,
+            exception,
+            timestamp: Date.now(),
+        });
+    }
+
+    public dumpMessages(): string {
+        return this.records
+            .flatMap(({ level, message, timestamp, exception }) => {
+                const dateStr = new Date(timestamp).toISOString();
+                const lines = [`[${dateStr} - ${level}] ${message}`];
+                if (exception !== undefined) lines.push(`${exception}`);
+                return lines;
+            })
+            .join('\n');
+    }
+
+    public readonly onDebug = this.saveMessage.bind(this, 'DEBUG');
+    public readonly onLog = this.saveMessage.bind(this, 'LOG');
+    public readonly onInfo = this.saveMessage.bind(this, 'INFO');
+    public readonly onSuccess = this.saveMessage.bind(this, 'SUCCESS');
+    public readonly onWarn = this.saveMessage.bind(this, 'WARNING');
+    public readonly onError = this.saveMessage.bind(this, 'ERROR');
+
+    public readonly minimumLevel = LogLevel.DEBUG;
+}

src/lib/logging/logger.ts

@@ -6,14 +6,14 @@ interface LoggerOptions {
     sinks: LoggingSink[];
 }
 
-const HANDLER_NAMES: Record<LogLevel, keyof LoggingSink> = {
+const HANDLER_NAMES = {
     [LogLevel.DEBUG]: 'onDebug',
     [LogLevel.LOG]: 'onLog',
     [LogLevel.INFO]: 'onInfo',
     [LogLevel.SUCCESS]: 'onSuccess',
     [LogLevel.WARNING]: 'onWarn',
     [LogLevel.ERROR]: 'onError',
-};
+} as const;
 
 const DEFAULT_OPTIONS = {
     logLevel: LogLevel.INFO,
@@ -31,10 +31,11 @@ export class Logger {
     }
 
     private fireHandlers(level: LogLevel, message: string, exception?: unknown): void {
-        if (level < this._configuration.logLevel) return;
-
         this._configuration.sinks
             .forEach((sink) => {
+                const minLevel = sink.minimumLevel ?? this.configuration.logLevel;
+                if (level < minLevel) return;
+
                 const handler = sink[HANDLER_NAMES[level]];
                 if (!handler) return;
 

src/lib/logging/sink.ts

@@ -1,8 +1,16 @@
+import type { LogLevel } from './levels';
+
 export interface LoggingSink {
     onDebug?(message: string): void;
     onLog?(message: string): void;
     onInfo?(message: string): void;
     onSuccess?(message: string): void;
     onWarn?(message: string, exception?: unknown): void;
     onError?(message: string, exception?: unknown): void;
+
+    /**
+     * Minimum level of log messages to pass to this sink. If left undefined,
+     * the logger will use the minimum level set on the logger itself.
+     */
+    minimumLevel?: LogLevel;
 }

src/lib/util/array.ts

@@ -67,3 +67,7 @@ export function insertBetween<T1, T2>(arr: readonly T1[], newElement: T2 | (() =
         ...arr.slice(1).flatMap((elmt) => [isFactory(newElement) ? newElement() : newElement, elmt]),
     ];
 }
+
+export function deduplicateArray(arr: readonly string[]): string[] {
+    return [...new Set(arr)];
+}

src/lib/util/request/index.ts

@@ -1,5 +1,6 @@
 /* eslint-disable no-restricted-globals */
 
+import type { RequestObserver } from './observers';
 import type { RequestMethod, RequestOptions } from './requestOptions';
 import type { Response, ResponseFor, TextResponse } from './response';
 import { performFetchRequest } from './backendFetch';
@@ -25,6 +26,8 @@ interface RequestFunc {
 
     head<RequestOptionsT extends RequestOptions>(url: string | URL, options: RequestOptionsT): Promise<ResponseFor<RequestOptionsT>>;
     head(url: string | URL): Promise<TextResponse>;
+
+    addObserver(observer: RequestObserver): void;
 }
 
 const hasGMXHR = (
@@ -33,22 +36,65 @@ const hasGMXHR = (
     // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Might be using GMv3 API.
     || (typeof GM !== 'undefined' && GM.xmlHttpRequest !== undefined));
 
-export const request: RequestFunc = async function (method: RequestMethod, url: string | URL, options?: RequestOptions) {
-    // istanbul ignore next: Difficult to test.
-    const backend = options?.backend ?? (hasGMXHR ? RequestBackend.GMXHR : RequestBackend.FETCH);
-    const response = await performRequest(backend, method, url, options);
+export const request = ((): RequestFunc => {
+    const observers: RequestObserver[] = [];
+
+    function notifyObservers<EventT extends keyof RequestObserver>(event: EventT, data: Parameters<NonNullable<RequestObserver[EventT]>>[0]): void {
+        for (const observer of observers) {
+            // @ts-expect-error: False positive?
+            observer[event]?.(data);
+        }
+    }
 
-    const throwForStatus = options?.throwForStatus ?? true;
-    if (throwForStatus && response.status >= 400) {
-        throw new HTTPResponseError(url, response, options?.httpErrorMessages?.[response.status]);
+    function insertDefaultProgressListener(backend: RequestBackend, method: RequestMethod, url: URL | string, options?: RequestOptions): RequestOptions {
+        return {
+            ...options,
+            // istanbul ignore next: Difficult to cover, test gmxhr doesn't emit progress events.
+            onProgress: (progressEvent): void => {
+                notifyObservers('onProgress', { backend, method, url, options, progressEvent });
+                // Also pass through this progress event to original listener if it exists.
+                options?.onProgress?.(progressEvent);
+            },
+        };
     }
 
-    return response;
-} as RequestFunc;
+    const impl = async function (method: RequestMethod, url: string | URL, options?: RequestOptions) {
+        // istanbul ignore next: Difficult to test.
+        const backend = options?.backend ?? (hasGMXHR ? RequestBackend.GMXHR : RequestBackend.FETCH);
+
+        try {
+            notifyObservers('onStarted', { backend, method, url, options });
+
+            // Inject own progress listener so we can echo that to the observers.
+            const optionsWithProgressWrapper = insertDefaultProgressListener(backend, method, url, options);
+            const response = await performRequest(backend, method, url, optionsWithProgressWrapper);
+
+            const throwForStatus = options?.throwForStatus ?? true;
+            if (throwForStatus && response.status >= 400) {
+                throw new HTTPResponseError(url, response, options?.httpErrorMessages?.[response.status]);
+            }
+
+            notifyObservers('onSuccess', { backend, method, url, options, response });
+            return response;
+        } catch (err) {
+            // istanbul ignore else: Should not happen in practice.
+            if (err instanceof Error) {
+                notifyObservers('onFailed', { backend, method, url, options, error: err });
+            }
+            throw err;
+        }
+    } as RequestFunc;
+
+    impl.get = impl.bind(undefined, 'GET');
+    impl.post = impl.bind(undefined, 'POST');
+    impl.head = impl.bind(undefined, 'HEAD');
+
+    impl.addObserver = (observer): void => {
+        observers.push(observer);
+    };
 
-request.get = request.bind(undefined, 'GET');
-request.post = request.bind(undefined, 'POST');
-request.head = request.bind(undefined, 'HEAD');
+    return impl;
+})();
 
 function performRequest(backend: RequestBackend, method: RequestMethod, url: string | URL, options?: RequestOptions): Promise<Response> {
     switch (backend) {

src/lib/util/request/observers/index.ts

@@ -0,0 +1,3 @@
+export { loggingObserver } from './loggingObserver';
+export { RecordingObserver } from './recordingObserver';
+export type { RequestObserver } from './types';

src/lib/util/request/observers/loggingObserver.ts

@@ -0,0 +1,20 @@
+import { LOGGER } from '@lib/logging/logger';
+
+import type { RequestObserver } from './types';
+
+export const loggingObserver: RequestObserver = {
+    onStarted({ backend, method, url }) {
+        LOGGER.debug(`${method} ${url} - STARTED (backend: ${backend})`);
+    },
+    onSuccess({ method, url, response }) {
+        LOGGER.debug(`${method} ${url} - SUCCESS (code ${response.status})`);
+    },
+    onFailed({ method, url, error }) {
+        LOGGER.debug(`${method} ${url} - FAILED (${error})`);
+    },
+    // istanbul ignore next: Unit tests don't have progress events.
+    onProgress({ method, url, progressEvent }) {
+        const { loaded, total } = progressEvent;
+        LOGGER.debug(`${method} ${url} - PROGRESS (${loaded}/${total})`);
+    },
+};

src/lib/util/request/observers/recordingObserver.ts

@@ -0,0 +1,110 @@
+import { deduplicateArray } from '@lib/util/array';
+
+import type { ArrayBufferResponse, BlobResponse, Response, TextResponse } from '../response';
+import type { BaseRequestEvent, RequestObserver } from './types';
+import { HTTPResponseError } from '../errors';
+
+type RecordedResponse = Omit<TextResponse, 'json' | 'rawResponse'>;
+interface Recording {
+    requestInfo: BaseRequestEvent;
+    response: RecordedResponse;
+}
+
+/**
+ * Convert a response to a textual one. For blob and arraybuffer, strips the
+ * actual content and sets text to a placeholder string. We don't record the
+ * original responses since they can be large and would cause memory leaks.
+ */
+function convertResponse(response: Response): RecordedResponse {
+    if (Object.prototype.hasOwnProperty.call(response, 'text')) return response as TextResponse;
+
+    const text = Object.prototype.hasOwnProperty.call(response, 'blob')
+        ? `<Blob, ${(response as BlobResponse).blob.size} bytes>`
+        : `<ArrayBuffer, ${(response as ArrayBufferResponse).arrayBuffer.byteLength} bytes>`;
+
+    return {
+        headers: response.headers,
+        status: response.status,
+        statusText: response.statusText,
+        url: response.url,
+        text,
+    };
+}
+
+/**
+ * Convert request info for recording. Strips down the passed in object to
+ * remove references to responses to prevent memory leaks. Returns a copy.
+ */
+function convertRequestInfo(requestInfo: BaseRequestEvent): BaseRequestEvent {
+    return {
+        backend: requestInfo.backend,
+        url: requestInfo.url,
+        method: requestInfo.method,
+        options: requestInfo.options,
+    };
+}
+
+function getURLHost(url: string | URL): string {
+    return new URL(url).host;
+}
+
+function exportRecordedResponse(recordedResponse: Recording): string {
+    const { requestInfo, response } = recordedResponse;
+    const { backend, method, url, options: reqOptions } = requestInfo;
+
+    const reqOptionsString = JSON.stringify(reqOptions, (key, value) => {
+        // Don't include the progress callback.
+        if (key === 'onProgress') return undefined;
+        // eslint-disable-next-line @typescript-eslint/no-unsafe-return
+        return value;
+    }, 2);
+    const reqPreamble = `${method} ${url} (backend: ${backend})\nOptions: ${reqOptionsString}`;
+
+    const respPreamble = `${response.url} ${response.status}: ${response.statusText}`;
+    const respHeaders = [...response.headers.entries()]
+        .map(([name, value]) => `${name}: ${value}`)
+        .join('\n');
+
+    return [reqPreamble, '\n', respPreamble, respHeaders, '\n', response.text].join('\n');
+}
+
+export class RecordingObserver implements RequestObserver {
+    private readonly recordedResponses: Recording[];
+
+    public constructor() {
+        this.recordedResponses = [];
+    }
+
+    public onSuccess(event: BaseRequestEvent & { response: Response }): void {
+        this.recordedResponses.push({
+            requestInfo: convertRequestInfo(event),
+            response: convertResponse(event.response),
+        });
+    }
+
+    public onFailed(event: BaseRequestEvent & { error: Error }): void {
+        if (!(event.error instanceof HTTPResponseError)) return;
+        this.recordedResponses.push({
+            requestInfo: convertRequestInfo(event),
+            response: convertResponse(event.error.response),
+        });
+    }
+
+    public exportResponses(): string {
+        return this.recordedResponses
+            .map((recordedResponse) => exportRecordedResponse(recordedResponse))
+            .join('\n\n==============================\n\n');
+    }
+
+    public hasRecordings(): boolean {
+        return this.recordedResponses.length > 0;
+    }
+
+    public get recordedDomains(): string[] {
+        return deduplicateArray(this.recordedResponses.flatMap((rec) => {
+            const domains = [getURLHost(rec.requestInfo.url)];
+            if (rec.response.url) domains.push(getURLHost(rec.response.url));
+            return domains;
+        }));
+    }
+}

src/lib/util/request/observers/types.ts

@@ -0,0 +1,16 @@
+import type { RequestBackend, RequestMethod, RequestOptions } from '../requestOptions';
+import type { ProgressEvent, Response } from '../response';
+
+export interface BaseRequestEvent {
+    backend: RequestBackend;
+    method: RequestMethod;
+    url: URL | string;
+    options?: RequestOptions;
+}
+
+export interface RequestObserver {
+    onStarted?: (event: Readonly<BaseRequestEvent>) => void;
+    onFailed?: (event: Readonly<BaseRequestEvent & { error: Error }>) => void;
+    onSuccess?: (event: Readonly<BaseRequestEvent & { response: Response }>) => void;
+    onProgress?: (event: Readonly<BaseRequestEvent & { progressEvent: ProgressEvent }>) => void;
+}