diff --git a/build/rollup.ts b/build/rollup.ts index 9a270068..104c0c3d 100644 --- a/build/rollup.ts +++ b/build/rollup.ts @@ -15,6 +15,8 @@ import postcss from 'rollup-plugin-postcss'; import progress from 'rollup-plugin-progress'; import { minify } from 'terser'; +import { deduplicateArray } from '@lib/util/array'; + import { parseChangelogEntries } from './changelog'; import { consts } from './plugin-consts'; import { logger } from './plugin-logger'; @@ -249,7 +251,7 @@ function getVendorMinifiedPreamble(chunk: Readonly): string { .slice(0, module.startsWith('@') ? 2 : 1) .join('/')); - const uniqueBundledModules = [...new Set(bundledModules)]; + const uniqueBundledModules = deduplicateArray(bundledModules); if ('\u0000rollupPluginBabelHelpers.js' in chunk.modules) { uniqueBundledModules.unshift('babel helpers'); } diff --git a/src/lib/MB/URLs.ts b/src/lib/MB/URLs.ts index 85679018..63a0e72f 100644 --- a/src/lib/MB/URLs.ts +++ b/src/lib/MB/URLs.ts @@ -1,3 +1,4 @@ +import { deduplicateArray } from '@lib/util/array'; import { request } from '@lib/util/request'; import type { ReleaseAdvRel, URLAdvRel } from './advanced-relationships'; @@ -21,7 +22,7 @@ export async function getURLsForRelease(releaseId: string, options?: { excludeEn } let urls = urlARs.map((ar) => ar.url.resource); if (excludeDuplicates) { - urls = [...new Set(urls)]; + urls = deduplicateArray(urls); } return urls.flatMap((url) => { diff --git a/src/lib/logging/collectorSink.ts b/src/lib/logging/collectorSink.ts new file mode 100644 index 00000000..05a77eeb --- /dev/null +++ b/src/lib/logging/collectorSink.ts @@ -0,0 +1,46 @@ +import type { LoggingSink } from './sink'; +import { LogLevel } from './levels'; + +interface LogRecord { + level: string; + message: string; + timestamp: number; + exception?: unknown; +} + +export class CollectorSink implements LoggingSink { + private readonly records: LogRecord[]; + + public constructor() { + this.records = []; + } + + private saveMessage(level: string, message: string, exception?: unknown): void { + this.records.push({ + level, + message, + exception, + timestamp: Date.now(), + }); + } + + public dumpMessages(): string { + return this.records + .flatMap(({ level, message, timestamp, exception }) => { + const dateStr = new Date(timestamp).toISOString(); + const lines = [`[${dateStr} - ${level}] ${message}`]; + if (exception !== undefined) lines.push(`${exception}`); + return lines; + }) + .join('\n'); + } + + public readonly onDebug = this.saveMessage.bind(this, 'DEBUG'); + public readonly onLog = this.saveMessage.bind(this, 'LOG'); + public readonly onInfo = this.saveMessage.bind(this, 'INFO'); + public readonly onSuccess = this.saveMessage.bind(this, 'SUCCESS'); + public readonly onWarn = this.saveMessage.bind(this, 'WARNING'); + public readonly onError = this.saveMessage.bind(this, 'ERROR'); + + public readonly minimumLevel = LogLevel.DEBUG; +} diff --git a/src/lib/logging/logger.ts b/src/lib/logging/logger.ts index dc9dbc4b..4a827c24 100644 --- a/src/lib/logging/logger.ts +++ b/src/lib/logging/logger.ts @@ -6,14 +6,14 @@ interface LoggerOptions { sinks: LoggingSink[]; } -const HANDLER_NAMES: Record = { +const HANDLER_NAMES = { [LogLevel.DEBUG]: 'onDebug', [LogLevel.LOG]: 'onLog', [LogLevel.INFO]: 'onInfo', [LogLevel.SUCCESS]: 'onSuccess', [LogLevel.WARNING]: 'onWarn', [LogLevel.ERROR]: 'onError', -}; +} as const; const DEFAULT_OPTIONS = { logLevel: LogLevel.INFO, @@ -31,10 +31,11 @@ export class Logger { } private fireHandlers(level: LogLevel, message: string, exception?: unknown): void { - if (level < this._configuration.logLevel) return; - this._configuration.sinks .forEach((sink) => { + const minLevel = sink.minimumLevel ?? this.configuration.logLevel; + if (level < minLevel) return; + const handler = sink[HANDLER_NAMES[level]]; if (!handler) return; diff --git a/src/lib/logging/sink.ts b/src/lib/logging/sink.ts index 53eb035d..6dbab275 100644 --- a/src/lib/logging/sink.ts +++ b/src/lib/logging/sink.ts @@ -1,3 +1,5 @@ +import type { LogLevel } from './levels'; + export interface LoggingSink { onDebug?(message: string): void; onLog?(message: string): void; @@ -5,4 +7,10 @@ export interface LoggingSink { onSuccess?(message: string): void; onWarn?(message: string, exception?: unknown): void; onError?(message: string, exception?: unknown): void; + + /** + * Minimum level of log messages to pass to this sink. If left undefined, + * the logger will use the minimum level set on the logger itself. + */ + minimumLevel?: LogLevel; } diff --git a/src/lib/util/array.ts b/src/lib/util/array.ts index 20c068fa..d3cafeab 100644 --- a/src/lib/util/array.ts +++ b/src/lib/util/array.ts @@ -67,3 +67,7 @@ export function insertBetween(arr: readonly T1[], newElement: T2 | (() = ...arr.slice(1).flatMap((elmt) => [isFactory(newElement) ? newElement() : newElement, elmt]), ]; } + +export function deduplicateArray(arr: readonly string[]): string[] { + return [...new Set(arr)]; +} diff --git a/src/lib/util/request/index.ts b/src/lib/util/request/index.ts index 1a3aad53..d553d53b 100644 --- a/src/lib/util/request/index.ts +++ b/src/lib/util/request/index.ts @@ -1,5 +1,6 @@ /* eslint-disable no-restricted-globals */ +import type { RequestObserver } from './observers'; import type { RequestMethod, RequestOptions } from './requestOptions'; import type { Response, ResponseFor, TextResponse } from './response'; import { performFetchRequest } from './backendFetch'; @@ -25,6 +26,8 @@ interface RequestFunc { head(url: string | URL, options: RequestOptionsT): Promise>; head(url: string | URL): Promise; + + addObserver(observer: RequestObserver): void; } const hasGMXHR = ( @@ -33,22 +36,65 @@ const hasGMXHR = ( // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition -- Might be using GMv3 API. || (typeof GM !== 'undefined' && GM.xmlHttpRequest !== undefined)); -export const request: RequestFunc = async function (method: RequestMethod, url: string | URL, options?: RequestOptions) { - // istanbul ignore next: Difficult to test. - const backend = options?.backend ?? (hasGMXHR ? RequestBackend.GMXHR : RequestBackend.FETCH); - const response = await performRequest(backend, method, url, options); +export const request = ((): RequestFunc => { + const observers: RequestObserver[] = []; + + function notifyObservers(event: EventT, data: Parameters>[0]): void { + for (const observer of observers) { + // @ts-expect-error: False positive? + observer[event]?.(data); + } + } - const throwForStatus = options?.throwForStatus ?? true; - if (throwForStatus && response.status >= 400) { - throw new HTTPResponseError(url, response, options?.httpErrorMessages?.[response.status]); + function insertDefaultProgressListener(backend: RequestBackend, method: RequestMethod, url: URL | string, options?: RequestOptions): RequestOptions { + return { + ...options, + // istanbul ignore next: Difficult to cover, test gmxhr doesn't emit progress events. + onProgress: (progressEvent): void => { + notifyObservers('onProgress', { backend, method, url, options, progressEvent }); + // Also pass through this progress event to original listener if it exists. + options?.onProgress?.(progressEvent); + }, + }; } - return response; -} as RequestFunc; + const impl = async function (method: RequestMethod, url: string | URL, options?: RequestOptions) { + // istanbul ignore next: Difficult to test. + const backend = options?.backend ?? (hasGMXHR ? RequestBackend.GMXHR : RequestBackend.FETCH); + + try { + notifyObservers('onStarted', { backend, method, url, options }); + + // Inject own progress listener so we can echo that to the observers. + const optionsWithProgressWrapper = insertDefaultProgressListener(backend, method, url, options); + const response = await performRequest(backend, method, url, optionsWithProgressWrapper); + + const throwForStatus = options?.throwForStatus ?? true; + if (throwForStatus && response.status >= 400) { + throw new HTTPResponseError(url, response, options?.httpErrorMessages?.[response.status]); + } + + notifyObservers('onSuccess', { backend, method, url, options, response }); + return response; + } catch (err) { + // istanbul ignore else: Should not happen in practice. + if (err instanceof Error) { + notifyObservers('onFailed', { backend, method, url, options, error: err }); + } + throw err; + } + } as RequestFunc; + + impl.get = impl.bind(undefined, 'GET'); + impl.post = impl.bind(undefined, 'POST'); + impl.head = impl.bind(undefined, 'HEAD'); + + impl.addObserver = (observer): void => { + observers.push(observer); + }; -request.get = request.bind(undefined, 'GET'); -request.post = request.bind(undefined, 'POST'); -request.head = request.bind(undefined, 'HEAD'); + return impl; +})(); function performRequest(backend: RequestBackend, method: RequestMethod, url: string | URL, options?: RequestOptions): Promise { switch (backend) { diff --git a/src/lib/util/request/observers/index.ts b/src/lib/util/request/observers/index.ts new file mode 100644 index 00000000..87084e45 --- /dev/null +++ b/src/lib/util/request/observers/index.ts @@ -0,0 +1,3 @@ +export { loggingObserver } from './loggingObserver'; +export { RecordingObserver } from './recordingObserver'; +export type { RequestObserver } from './types'; diff --git a/src/lib/util/request/observers/loggingObserver.ts b/src/lib/util/request/observers/loggingObserver.ts new file mode 100644 index 00000000..350cb142 --- /dev/null +++ b/src/lib/util/request/observers/loggingObserver.ts @@ -0,0 +1,20 @@ +import { LOGGER } from '@lib/logging/logger'; + +import type { RequestObserver } from './types'; + +export const loggingObserver: RequestObserver = { + onStarted({ backend, method, url }) { + LOGGER.debug(`${method} ${url} - STARTED (backend: ${backend})`); + }, + onSuccess({ method, url, response }) { + LOGGER.debug(`${method} ${url} - SUCCESS (code ${response.status})`); + }, + onFailed({ method, url, error }) { + LOGGER.debug(`${method} ${url} - FAILED (${error})`); + }, + // istanbul ignore next: Unit tests don't have progress events. + onProgress({ method, url, progressEvent }) { + const { loaded, total } = progressEvent; + LOGGER.debug(`${method} ${url} - PROGRESS (${loaded}/${total})`); + }, +}; diff --git a/src/lib/util/request/observers/recordingObserver.ts b/src/lib/util/request/observers/recordingObserver.ts new file mode 100644 index 00000000..1a06c0b4 --- /dev/null +++ b/src/lib/util/request/observers/recordingObserver.ts @@ -0,0 +1,110 @@ +import { deduplicateArray } from '@lib/util/array'; + +import type { ArrayBufferResponse, BlobResponse, Response, TextResponse } from '../response'; +import type { BaseRequestEvent, RequestObserver } from './types'; +import { HTTPResponseError } from '../errors'; + +type RecordedResponse = Omit; +interface Recording { + requestInfo: BaseRequestEvent; + response: RecordedResponse; +} + +/** + * Convert a response to a textual one. For blob and arraybuffer, strips the + * actual content and sets text to a placeholder string. We don't record the + * original responses since they can be large and would cause memory leaks. + */ +function convertResponse(response: Response): RecordedResponse { + if (Object.prototype.hasOwnProperty.call(response, 'text')) return response as TextResponse; + + const text = Object.prototype.hasOwnProperty.call(response, 'blob') + ? `` + : ``; + + return { + headers: response.headers, + status: response.status, + statusText: response.statusText, + url: response.url, + text, + }; +} + +/** + * Convert request info for recording. Strips down the passed in object to + * remove references to responses to prevent memory leaks. Returns a copy. + */ +function convertRequestInfo(requestInfo: BaseRequestEvent): BaseRequestEvent { + return { + backend: requestInfo.backend, + url: requestInfo.url, + method: requestInfo.method, + options: requestInfo.options, + }; +} + +function getURLHost(url: string | URL): string { + return new URL(url).host; +} + +function exportRecordedResponse(recordedResponse: Recording): string { + const { requestInfo, response } = recordedResponse; + const { backend, method, url, options: reqOptions } = requestInfo; + + const reqOptionsString = JSON.stringify(reqOptions, (key, value) => { + // Don't include the progress callback. + if (key === 'onProgress') return undefined; + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return value; + }, 2); + const reqPreamble = `${method} ${url} (backend: ${backend})\nOptions: ${reqOptionsString}`; + + const respPreamble = `${response.url} ${response.status}: ${response.statusText}`; + const respHeaders = [...response.headers.entries()] + .map(([name, value]) => `${name}: ${value}`) + .join('\n'); + + return [reqPreamble, '\n', respPreamble, respHeaders, '\n', response.text].join('\n'); +} + +export class RecordingObserver implements RequestObserver { + private readonly recordedResponses: Recording[]; + + public constructor() { + this.recordedResponses = []; + } + + public onSuccess(event: BaseRequestEvent & { response: Response }): void { + this.recordedResponses.push({ + requestInfo: convertRequestInfo(event), + response: convertResponse(event.response), + }); + } + + public onFailed(event: BaseRequestEvent & { error: Error }): void { + if (!(event.error instanceof HTTPResponseError)) return; + this.recordedResponses.push({ + requestInfo: convertRequestInfo(event), + response: convertResponse(event.error.response), + }); + } + + public exportResponses(): string { + return this.recordedResponses + .map((recordedResponse) => exportRecordedResponse(recordedResponse)) + .join('\n\n==============================\n\n'); + } + + public hasRecordings(): boolean { + return this.recordedResponses.length > 0; + } + + public get recordedDomains(): string[] { + return deduplicateArray(this.recordedResponses.flatMap((rec) => { + const domains = [getURLHost(rec.requestInfo.url)]; + if (rec.response.url) domains.push(getURLHost(rec.response.url)); + return domains; + })); + } +} diff --git a/src/lib/util/request/observers/types.ts b/src/lib/util/request/observers/types.ts new file mode 100644 index 00000000..bb8853cb --- /dev/null +++ b/src/lib/util/request/observers/types.ts @@ -0,0 +1,16 @@ +import type { RequestBackend, RequestMethod, RequestOptions } from '../requestOptions'; +import type { ProgressEvent, Response } from '../response'; + +export interface BaseRequestEvent { + backend: RequestBackend; + method: RequestMethod; + url: URL | string; + options?: RequestOptions; +} + +export interface RequestObserver { + onStarted?: (event: Readonly) => void; + onFailed?: (event: Readonly) => void; + onSuccess?: (event: Readonly) => void; + onProgress?: (event: Readonly) => void; +} diff --git a/src/mb_enhanced_cover_art_uploads/App.ts b/src/mb_enhanced_cover_art_uploads/App.ts index fad57935..b923cc9d 100644 --- a/src/mb_enhanced_cover_art_uploads/App.ts +++ b/src/mb_enhanced_cover_art_uploads/App.ts @@ -1,5 +1,6 @@ /* istanbul ignore file: Covered by E2E */ +import { CollectorSink } from '@lib/logging/collectorSink'; import { GuiSink } from '@lib/logging/guiSink'; import { LOGGER } from '@lib/logging/logger'; import { EditNote } from '@lib/MB/EditNote'; @@ -9,6 +10,8 @@ import { assertHasValue } from '@lib/util/assert'; import { pFinally } from '@lib/util/async'; import { qs } from '@lib/util/dom'; import { ObservableSemaphore } from '@lib/util/observable'; +import { request } from '@lib/util/request'; +import { loggingObserver, RecordingObserver } from '@lib/util/request/observers'; import type { BareCoverArt, QueuedImageBatch } from './types'; import { ImageFetcher } from './fetch'; @@ -23,6 +26,8 @@ export class App { private readonly ui: InputForm; private readonly urlsInProgress: Set; private readonly loggingSink = new GuiSink(); + private readonly collectorSink = new CollectorSink(); + private readonly requestRecorder = new RecordingObserver(); private readonly fetchingSema: ObservableSemaphore; public onlyFront = false; @@ -30,6 +35,13 @@ export class App { this.note = EditNote.withFooterFromGMInfo(); this.urlsInProgress = new Set(); + // Set up log collector + LOGGER.addSink(this.collectorSink); + + // Set up request recorder + request.addObserver(loggingObserver); + request.addObserver(this.requestRecorder); + // Set up logging banner LOGGER.addSink(this.loggingSink); qs('.add-files').insertAdjacentElement('afterend', this.loggingSink.rootElement); @@ -133,4 +145,33 @@ export class App { return this.ui.addImportButton(syncProcessURL.bind(this, url), url.href, provider); })); } + + private confirmExportRequests(): boolean { + const msg = [ + 'Recorded requests contain responses from the following domains:\n', + this.requestRecorder.recordedDomains.join(', ') + '\n', + 'Exporting the request responses may leak user credentials and keys if you are logged in to any of these domains! ', + 'Do you want to include the responses in the exported debug content?', + ].join(''); + return window.confirm(msg); + } + + public exportDebugLogs(): void { + let exportedContent = this.collectorSink.dumpMessages(); + + if (this.requestRecorder.hasRecordings() && this.confirmExportRequests()) { + exportedContent += '\n\n==============================\n\n'; + exportedContent += this.requestRecorder.exportResponses(); + } + + // Download the file by creating a fake anchor and clicking it. + const downloadAnchor = document.createElement('a'); + const downloadUrl = URL.createObjectURL(new Blob([exportedContent], { type: 'text/plain' })); + downloadAnchor.download = `ECAU-debug-${Date.now()}.txt`; + downloadAnchor.href = downloadUrl; + downloadAnchor.click(); + setTimeout(() => { + URL.revokeObjectURL(downloadUrl); + }); + } } diff --git a/src/mb_enhanced_cover_art_uploads/ui/main.tsx b/src/mb_enhanced_cover_art_uploads/ui/main.tsx index 39619e02..5e61aabb 100644 --- a/src/mb_enhanced_cover_art_uploads/ui/main.tsx +++ b/src/mb_enhanced_cover_art_uploads/ui/main.tsx @@ -156,6 +156,7 @@ export class InputForm implements FetcherHooks { // Container element for the URL input and additional information const container =
{this.urlInput} + Export debug logs -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:43.941Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:0aec3abfd05cd731dfe443bbc71c853dd2c9eb023a4ed6471ac06d0351cf6cd9 -Content-Length: 152 - -GET /base64/aGVsbG8gd29ybGQ= HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:43.941Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:1adb376bd0b881c4a6d9ca3443cfbdd7a2a3638cde864459562c151f310256c1 -WARC-Block-Digest: sha256:1adb376bd0b881c4a6d9ca3443cfbdd7a2a3638cde864459562c151f310256c1 -Content-Length: 349 - -harEntryId: 75b5494019dca4a878b1981d08716089 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T18:00:43.398Z -time: 535 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":535,"receive":0,"ssl":-1} -warcRequestHeadersSize: 171 -warcRequestCookies: [] -warcResponseHeadersSize: 215 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:43.941Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:7caa0461a2dd0b11ac0a4b3d4e6c76fd5b2695e83eb1bc7ff451c64b79c8ca49 -Content-Length: 243 - -HTTP/1.1 200 OK -date: Thu, 11 Aug 2022 18:00:43 GMT -content-type: text/html; charset=utf-8 -content-length: 11 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - -hello world - diff --git a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/defaults-to-textual-responses_3408381847.warc b/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/defaults-to-textual-responses_3408381847.warc deleted file mode 100644 index c2261bde..00000000 --- a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/defaults-to-textual-responses_3408381847.warc +++ /dev/null @@ -1,78 +0,0 @@ -WARC/1.1 -WARC-Filename: request/fetch backend/defaults to textual responses -WARC-Date: 2022-08-11T17:54:49.702Z -WARC-Type: warcinfo -WARC-Record-ID: -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:54:49.703Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:0aec3abfd05cd731dfe443bbc71c853dd2c9eb023a4ed6471ac06d0351cf6cd9 -Content-Length: 152 - -GET /base64/aGVsbG8gd29ybGQ= HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:54:49.703Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:39a347a4dd548b516997aa0c678bd9c3f0932b49f73cf21fd27d6290bd76be4e -WARC-Block-Digest: sha256:39a347a4dd548b516997aa0c678bd9c3f0932b49f73cf21fd27d6290bd76be4e -Content-Length: 349 - -harEntryId: 75b5494019dca4a878b1981d08716089 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T17:54:49.135Z -time: 559 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":559,"receive":0,"ssl":-1} -warcRequestHeadersSize: 171 -warcRequestCookies: [] -warcResponseHeadersSize: 215 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:54:49.702Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:48e06f49eb780ae00f7595403e1d47f7b6c9f7e4f9141791f73bf26baf232ae9 -Content-Length: 243 - -HTTP/1.1 200 OK -date: Thu, 11 Aug 2022 17:54:49 GMT -content-type: text/html; charset=utf-8 -content-length: 11 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - -hello world - diff --git a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/includes-JSON-parser-in-textual-responses_3773065511.warc b/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/includes-JSON-parser-in-textual-responses_3773065511.warc deleted file mode 100644 index 80eacfcb..00000000 --- a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/includes-JSON-parser-in-textual-responses_3773065511.warc +++ /dev/null @@ -1,99 +0,0 @@ -WARC/1.1 -WARC-Filename: request/fetch backend/includes JSON parser in textual responses -WARC-Date: 2022-08-11T18:02:24.170Z -WARC-Type: warcinfo -WARC-Record-ID: -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.171Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:a76c54a8b1ed0588fa05c76be7f6ef5c6c8feeb857b0f9cdc5bb746f7a9fe6a5 -Content-Length: 133 - -GET /json HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.171Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:544c8c3b29eed00634113023895655c24ddf5703ff886993746ccc2230f8b501 -WARC-Block-Digest: sha256:544c8c3b29eed00634113023895655c24ddf5703ff886993746ccc2230f8b501 -Content-Length: 349 - -harEntryId: 934cef95e7505987bec5a0f4e6519d90 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T18:02:23.619Z -time: 545 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":545,"receive":0,"ssl":-1} -warcRequestHeadersSize: 152 -warcRequestCookies: [] -warcResponseHeadersSize: 208 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.171Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:bab3dfd2a6c992e4bf589eee05fc9650cc9d6988660b947a7a87b99420d108f9 -WARC-Block-Digest: sha256:396f17be56dc8e7980fcccfd6dc30189a5390927544a75e395919dcb981b3b80 -Content-Length: 654 - -HTTP/1.1 200 OK -date: Thu, 11 Aug 2022 18:02:24 GMT -content-type: application/json -content-length: 429 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - -{ - "slideshow": { - "author": "Yours Truly", - "date": "date of publication", - "slides": [ - { - "title": "Wake up to WonderWidgets!", - "type": "all" - }, - { - "items": [ - "Why WonderWidgets are great", - "Who buys WonderWidgets" - ], - "title": "Overview", - "type": "all" - } - ], - "title": "Sample Slide Show" - } -} - - diff --git a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-arraybuffer-responseType-requests_3336440301.warc b/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-arraybuffer-responseType-requests_3336440301.warc deleted file mode 100644 index f655f2c8..00000000 --- a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-arraybuffer-responseType-requests_3336440301.warc +++ /dev/null @@ -1,78 +0,0 @@ -WARC/1.1 -WARC-Filename: request/fetch backend/supports arraybuffer responseType requests -WARC-Date: 2022-08-11T17:47:06.022Z -WARC-Type: warcinfo -WARC-Record-ID: -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:06.023Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:0aec3abfd05cd731dfe443bbc71c853dd2c9eb023a4ed6471ac06d0351cf6cd9 -Content-Length: 152 - -GET /base64/aGVsbG8gd29ybGQ= HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:06.023Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:2f8b9c8464da50ebb0f29fda873aef019f26051247afe8ec1d90aceb0bea8042 -WARC-Block-Digest: sha256:2f8b9c8464da50ebb0f29fda873aef019f26051247afe8ec1d90aceb0bea8042 -Content-Length: 349 - -harEntryId: 75b5494019dca4a878b1981d08716089 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T17:47:05.664Z -time: 339 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":339,"receive":0,"ssl":-1} -warcRequestHeadersSize: 171 -warcRequestCookies: [] -warcResponseHeadersSize: 215 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:06.022Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:ed69b85bdd6ebaf26773c56406e94bfc681b45ded302424b554e5c83b252bedc -Content-Length: 243 - -HTTP/1.1 200 OK -date: Thu, 11 Aug 2022 17:47:06 GMT -content-type: text/html; charset=utf-8 -content-length: 11 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - -hello world - diff --git a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-blob-responseType-requests_422233135.warc b/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-blob-responseType-requests_422233135.warc deleted file mode 100644 index e214dc14..00000000 --- a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-blob-responseType-requests_422233135.warc +++ /dev/null @@ -1,78 +0,0 @@ -WARC/1.1 -WARC-Filename: request/fetch backend/supports blob responseType requests -WARC-Date: 2022-08-11T17:47:05.657Z -WARC-Type: warcinfo -WARC-Record-ID: -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:05.658Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:0aec3abfd05cd731dfe443bbc71c853dd2c9eb023a4ed6471ac06d0351cf6cd9 -Content-Length: 152 - -GET /base64/aGVsbG8gd29ybGQ= HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:05.658Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:4a37e6d60bacf85c57bdabfa1fcf1fb18217467b5e9ba7ee83847ebf24828373 -WARC-Block-Digest: sha256:4a37e6d60bacf85c57bdabfa1fcf1fb18217467b5e9ba7ee83847ebf24828373 -Content-Length: 349 - -harEntryId: 75b5494019dca4a878b1981d08716089 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T17:47:05.319Z -time: 332 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":332,"receive":0,"ssl":-1} -warcRequestHeadersSize: 171 -warcRequestCookies: [] -warcResponseHeadersSize: 215 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:05.657Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:191a64f6051a0e0d22acb14f1b56a7b9d5e25e8f6a914371c0924a80c4464e82 -Content-Length: 243 - -HTTP/1.1 200 OK -date: Thu, 11 Aug 2022 17:47:05 GMT -content-type: text/html; charset=utf-8 -content-length: 11 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - -hello world - diff --git a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-text-responseType-requests_2732982167.warc b/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-text-responseType-requests_2732982167.warc deleted file mode 100644 index efd8dac8..00000000 --- a/tests/test-data/__recordings__/request_1051330994/fetch-backend_2955150227/supports-text-responseType-requests_2732982167.warc +++ /dev/null @@ -1,78 +0,0 @@ -WARC/1.1 -WARC-Filename: request/fetch backend/supports text responseType requests -WARC-Date: 2022-08-11T17:47:05.307Z -WARC-Type: warcinfo -WARC-Record-ID: -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:05.308Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:0aec3abfd05cd731dfe443bbc71c853dd2c9eb023a4ed6471ac06d0351cf6cd9 -Content-Length: 152 - -GET /base64/aGVsbG8gd29ybGQ= HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:05.308Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:a4c4fab666fd2672b10c23694c455f94d47c72bc7c9bffd8665438ca653bcf4b -WARC-Block-Digest: sha256:a4c4fab666fd2672b10c23694c455f94d47c72bc7c9bffd8665438ca653bcf4b -Content-Length: 349 - -harEntryId: 75b5494019dca4a878b1981d08716089 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T17:47:04.717Z -time: 576 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":576,"receive":0,"ssl":-1} -warcRequestHeadersSize: 171 -warcRequestCookies: [] -warcResponseHeadersSize: 215 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:47:05.307Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:191a64f6051a0e0d22acb14f1b56a7b9d5e25e8f6a914371c0924a80c4464e82 -Content-Length: 243 - -HTTP/1.1 200 OK -date: Thu, 11 Aug 2022 17:47:05 GMT -content-type: text/html; charset=utf-8 -content-length: 11 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - -hello world - diff --git a/tests/test-data/__recordings__/request_1051330994/frontend_3860412285/does-not-reject-if-disabled-by-caller_2744883421.warc b/tests/test-data/__recordings__/request_1051330994/frontend_3860412285/does-not-reject-if-disabled-by-caller_2744883421.warc deleted file mode 100644 index 08f83f7c..00000000 --- a/tests/test-data/__recordings__/request_1051330994/frontend_3860412285/does-not-reject-if-disabled-by-caller_2744883421.warc +++ /dev/null @@ -1,139 +0,0 @@ -WARC/1.1 -WARC-Filename: request/frontend/does not reject if disabled by caller -WARC-Date: 2022-08-11T18:16:32.742Z -WARC-Type: warcinfo -WARC-Record-ID: -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.742Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:a893948eb21d3dbec7e1a032e518fb041bef4fde63aee2f2753d9e8e107b9305 -Content-Length: 28 - -GET /status/404 HTTP/1.1 - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.742Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:50ea4925e7ef0b557ad4d1059006944b6fb87cb14b5c27007aedb9b2ebabfb25 -WARC-Block-Digest: sha256:50ea4925e7ef0b557ad4d1059006944b6fb87cb14b5c27007aedb9b2ebabfb25 -Content-Length: 348 - -harEntryId: 55b7d2fddaf85f12d5cd082798fdf191 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T18:16:32.398Z -time: 343 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":343,"receive":0,"ssl":-1} -warcRequestHeadersSize: 49 -warcRequestCookies: [] -warcResponseHeadersSize: 266 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.742Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:16e535a56374897c7e77c761a39eab2fff76e6b02a50f16b138de4c4899e2fc3 -Content-Length: 290 - -HTTP/1.1 404 Not Found -access-control-allow-credentials: true -access-control-allow-origin: * -connection: close -content-length: 0 -content-type: text/html; charset=utf-8 -date: Thu, 11 Aug 2022 18:16:32 GMT -server: gunicorn/19.9.0 -x-pollyjs-finalurl: https://httpbin.org/status/404 - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.742Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:5157e6b9b580a6c7e9cee7f2e3a20e2f13b8cdeee951b5e7473d56e9616b38d1 -Content-Length: 139 - -GET /status/404 HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.743Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:4a3c3941db11488b57fe8422ac6b5251a5667ad807c962846ebb8977732de944 -WARC-Block-Digest: sha256:4a3c3941db11488b57fe8422ac6b5251a5667ad807c962846ebb8977732de944 -Content-Length: 349 - -harEntryId: a504acfefdddde0737bdc537a8462565 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T18:16:32.399Z -time: 337 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":337,"receive":0,"ssl":-1} -warcRequestHeadersSize: 158 -warcRequestCookies: [] -warcResponseHeadersSize: 214 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.742Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:6d4910838f2c934093777fd591eb82f0bc8b982f2a2e65e9d1ded485384f5540 -Content-Length: 238 - -HTTP/1.1 404 Not Found -date: Thu, 11 Aug 2022 18:16:32 GMT -content-type: text/html; charset=utf-8 -content-length: 0 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - - - diff --git a/tests/test-data/__recordings__/request_1051330994/frontend_3860412285/rejects-with-custom-error-text-on-HTTP-errors_972536161.warc b/tests/test-data/__recordings__/request_1051330994/frontend_3860412285/rejects-with-custom-error-text-on-HTTP-errors_972536161.warc deleted file mode 100644 index 3bd9fcb1..00000000 --- a/tests/test-data/__recordings__/request_1051330994/frontend_3860412285/rejects-with-custom-error-text-on-HTTP-errors_972536161.warc +++ /dev/null @@ -1,139 +0,0 @@ -WARC/1.1 -WARC-Filename: request/frontend/rejects with custom error text on HTTP errors -WARC-Date: 2022-08-11T18:16:32.393Z -WARC-Type: warcinfo -WARC-Record-ID: -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.394Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:a893948eb21d3dbec7e1a032e518fb041bef4fde63aee2f2753d9e8e107b9305 -Content-Length: 28 - -GET /status/404 HTTP/1.1 - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.394Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:73223f60f5c70261ab77d7836ff737becf0b7e4ca0aaaca1074f0eb3653bd016 -WARC-Block-Digest: sha256:73223f60f5c70261ab77d7836ff737becf0b7e4ca0aaaca1074f0eb3653bd016 -Content-Length: 348 - -harEntryId: 55b7d2fddaf85f12d5cd082798fdf191 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T18:16:32.052Z -time: 338 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":338,"receive":0,"ssl":-1} -warcRequestHeadersSize: 49 -warcRequestCookies: [] -warcResponseHeadersSize: 266 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.393Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:16e535a56374897c7e77c761a39eab2fff76e6b02a50f16b138de4c4899e2fc3 -Content-Length: 290 - -HTTP/1.1 404 Not Found -access-control-allow-credentials: true -access-control-allow-origin: * -connection: close -content-length: 0 -content-type: text/html; charset=utf-8 -date: Thu, 11 Aug 2022 18:16:32 GMT -server: gunicorn/19.9.0 -x-pollyjs-finalurl: https://httpbin.org/status/404 - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.394Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:5157e6b9b580a6c7e9cee7f2e3a20e2f13b8cdeee951b5e7473d56e9616b38d1 -Content-Length: 139 - -GET /status/404 HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.394Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:43d8d6c6e58e3d04c8fdedf48c169714f020151277e7b74220b54bee244a0b6c -WARC-Block-Digest: sha256:43d8d6c6e58e3d04c8fdedf48c169714f020151277e7b74220b54bee244a0b6c -Content-Length: 349 - -harEntryId: a504acfefdddde0737bdc537a8462565 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T18:16:32.053Z -time: 335 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":335,"receive":0,"ssl":-1} -warcRequestHeadersSize: 158 -warcRequestCookies: [] -warcResponseHeadersSize: 214 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.394Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:6d4910838f2c934093777fd591eb82f0bc8b982f2a2e65e9d1ded485384f5540 -Content-Length: 238 - -HTTP/1.1 404 Not Found -date: Thu, 11 Aug 2022 18:16:32 GMT -content-type: text/html; charset=utf-8 -content-length: 0 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - - - diff --git a/tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/accepts-URL-input_4108038287.warc b/tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/accepts-URL-input_4108038287.warc deleted file mode 100644 index 9c97018f..00000000 --- a/tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/accepts-URL-input_4108038287.warc +++ /dev/null @@ -1,139 +0,0 @@ -WARC/1.1 -WARC-Filename: request/gmxhr backend/accepts URL input -WARC-Date: 2022-08-11T18:00:44.324Z -WARC-Type: warcinfo -WARC-Record-ID: -Content-Type: application/warc-fields -Content-Length: 119 - -software: warcio.js -harVersion: 1.2 -harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:44.324Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:5dc311a5e1c8cfd9601a019082ede513044c18fbc5054a3bee23585ea77e0c49 -Content-Length: 41 - -GET /base64/aGVsbG8gd29ybGQ= HTTP/1.1 - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:44.324Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:2f62d701c7191b1be40aa298494dba5e27ddfe0bab19b1742db6602bc9ba8074 -WARC-Block-Digest: sha256:2f62d701c7191b1be40aa298494dba5e27ddfe0bab19b1742db6602bc9ba8074 -Content-Length: 348 - -harEntryId: 11a31c2170ae4e633e3c914451961551 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T18:00:43.970Z -time: 352 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":352,"receive":0,"ssl":-1} -warcRequestHeadersSize: 62 -warcRequestCookies: [] -warcResponseHeadersSize: 280 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:44.324Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:6ef8792ff0e7bcd83bac3db3001419c13191be10999b97907c3c7d0e9bbf1c17 -Content-Length: 308 - -HTTP/1.1 200 OK -access-control-allow-credentials: true -access-control-allow-origin: * -connection: close -content-length: 11 -content-type: text/html; charset=utf-8 -date: Thu, 11 Aug 2022 18:00:44 GMT -server: gunicorn/19.9.0 -x-pollyjs-finalurl: https://httpbin.org/base64/aGVsbG8gd29ybGQ= - -hello world - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:44.324Z -WARC-Type: request -WARC-Record-ID: -Content-Type: application/http; msgtype=request -WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:0aec3abfd05cd731dfe443bbc71c853dd2c9eb023a4ed6471ac06d0351cf6cd9 -Content-Length: 152 - -GET /base64/aGVsbG8gd29ybGQ= HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org - - - -WARC/1.1 -WARC-Concurrent-To: -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:44.324Z -WARC-Type: metadata -WARC-Record-ID: -Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:190067811ad9cce55f6ad5f696937138604c0259cd03f10cc79d9bfed18b6c1e -WARC-Block-Digest: sha256:190067811ad9cce55f6ad5f696937138604c0259cd03f10cc79d9bfed18b6c1e -Content-Length: 349 - -harEntryId: 75b5494019dca4a878b1981d08716089 -harEntryOrder: 0 -cache: {} -startedDateTime: 2022-08-11T18:00:43.972Z -time: 347 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":347,"receive":0,"ssl":-1} -warcRequestHeadersSize: 171 -warcRequestCookies: [] -warcResponseHeadersSize: 215 -warcResponseCookies: [] -responseDecoded: false - - -WARC/1.1 -WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T18:00:44.324Z -WARC-Type: response -WARC-Record-ID: -Content-Type: application/http; msgtype=response -WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:bab6d8e6487b354844b3ddcbc61fac17218047f74c352b18a9ddddf1b8fad6f9 -Content-Length: 243 - -HTTP/1.1 200 OK -date: Thu, 11 Aug 2022 18:00:44 GMT -content-type: text/html; charset=utf-8 -content-length: 11 -connection: close -server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true - -hello world - diff --git a/tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/defaults-to-textual-responses_3408381847.warc b/tests/test-data/__recordings__/request_1051330994/hello-world_3582672807.warc similarity index 57% rename from tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/defaults-to-textual-responses_3408381847.warc rename to tests/test-data/__recordings__/request_1051330994/hello-world_3582672807.warc index d0045f6d..7cf12618 100644 --- a/tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/defaults-to-textual-responses_3408381847.warc +++ b/tests/test-data/__recordings__/request_1051330994/hello-world_3582672807.warc @@ -1,8 +1,8 @@ WARC/1.1 -WARC-Filename: request/gmxhr backend/defaults to textual responses -WARC-Date: 2022-08-11T17:56:01.014Z +WARC-Filename: request/hello world +WARC-Date: 2022-08-14T10:22:20.654Z WARC-Type: warcinfo -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields Content-Length: 119 @@ -12,11 +12,11 @@ harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:56:01.015Z +WARC-Date: 2022-08-14T10:22:20.654Z WARC-Type: request -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=request WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 WARC-Block-Digest: sha256:bcaba9cdb9e4152174015977e2b9529d4f2d8c2c8600f51977599ed114663221 @@ -32,22 +32,22 @@ user-agent: node-fetch WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:56:01.015Z +WARC-Date: 2022-08-14T10:22:20.654Z WARC-Type: metadata -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:a5a8d888693ab44c56aa5361be9c05620342a219f8ff458b047b465df94b370d -WARC-Block-Digest: sha256:a5a8d888693ab44c56aa5361be9c05620342a219f8ff458b047b465df94b370d +WARC-Payload-Digest: sha256:0473359b5bbbfc0d5b030af3e5213764a9505640281e7afb5bd74c17f714fdb7 +WARC-Block-Digest: sha256:0473359b5bbbfc0d5b030af3e5213764a9505640281e7afb5bd74c17f714fdb7 Content-Length: 349 harEntryId: 75b5494019dca4a878b1981d08716089 harEntryOrder: 0 cache: {} -startedDateTime: 2022-08-11T17:55:42.745Z -time: 651 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":651,"receive":0,"ssl":-1} +startedDateTime: 2022-08-14T10:22:19.163Z +time: 886 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":886,"receive":0,"ssl":-1} warcRequestHeadersSize: 171 warcRequestCookies: [] warcResponseHeadersSize: 215 @@ -57,12 +57,12 @@ responseDecoded: false WARC/1.1 WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:56:01.014Z +WARC-Date: 2022-08-14T10:22:20.654Z WARC-Type: response -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=response WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:d9095af1553ada44456b87830f2fce8425d4f5399540d87f5d4e7ecc714c4918 +WARC-Block-Digest: sha256:30eff511361be489505f88c5b5c165367607fce696bf2c5e5fa8e5fb2fb2fe22 Content-Length: 243 HTTP/1.1 200 OK @@ -71,17 +71,17 @@ access-control-allow-origin: * connection: close content-length: 11 content-type: text/html; charset=utf-8 -date: Thu, 11 Aug 2022 17:55:43 GMT +date: Sun, 14 Aug 2022 10:22:20 GMT server: gunicorn/19.9.0 hello world WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:56:01.015Z +WARC-Date: 2022-08-14T10:22:20.654Z WARC-Type: request -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=request WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 WARC-Block-Digest: sha256:5dc311a5e1c8cfd9601a019082ede513044c18fbc5054a3bee23585ea77e0c49 @@ -92,22 +92,22 @@ GET /base64/aGVsbG8gd29ybGQ= HTTP/1.1 WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:56:01.015Z +WARC-Date: 2022-08-14T10:22:20.654Z WARC-Type: metadata -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:a6e47cb6dc4de86b0aa63f111ab3148247f3097d58b142c5e887a0abdf603774 -WARC-Block-Digest: sha256:a6e47cb6dc4de86b0aa63f111ab3148247f3097d58b142c5e887a0abdf603774 -Content-Length: 346 +WARC-Payload-Digest: sha256:4dc00178a444931aa796c598c1ea930b4cdef2e1c0f2ea9e752f049497e2c4c7 +WARC-Block-Digest: sha256:4dc00178a444931aa796c598c1ea930b4cdef2e1c0f2ea9e752f049497e2c4c7 +Content-Length: 344 harEntryId: 11a31c2170ae4e633e3c914451961551 harEntryOrder: 0 cache: {} -startedDateTime: 2022-08-11T17:56:01.001Z -time: 10 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":10,"receive":0,"ssl":-1} +startedDateTime: 2022-08-14T10:22:20.645Z +time: 8 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":8,"receive":0,"ssl":-1} warcRequestHeadersSize: 62 warcRequestCookies: [] warcResponseHeadersSize: 280 @@ -117,12 +117,12 @@ responseDecoded: false WARC/1.1 WARC-Target-URI: https://httpbin.org/base64/aGVsbG8gd29ybGQ= -WARC-Date: 2022-08-11T17:56:01.015Z +WARC-Date: 2022-08-14T10:22:20.654Z WARC-Type: response -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=response WARC-Payload-Digest: sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9 -WARC-Block-Digest: sha256:d0f446afdfce500e1bc2ccb32ffa43361c59643c8f41bb370920eb0c7789f802 +WARC-Block-Digest: sha256:13c35610a64249ec323592b58a2e1494ce6df49f18ec3351145caa8845a0c669 Content-Length: 308 HTTP/1.1 200 OK @@ -131,7 +131,7 @@ access-control-allow-origin: * connection: close content-length: 11 content-type: text/html; charset=utf-8 -date: Thu, 11 Aug 2022 17:55:43 GMT +date: Sun, 14 Aug 2022 10:22:20 GMT server: gunicorn/19.9.0 x-pollyjs-finalurl: https://httpbin.org/base64/aGVsbG8gd29ybGQ= diff --git a/tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/includes-JSON-parser-in-textual-responses_3773065511.warc b/tests/test-data/__recordings__/request_1051330994/json_916562499.warc similarity index 60% rename from tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/includes-JSON-parser-in-textual-responses_3773065511.warc rename to tests/test-data/__recordings__/request_1051330994/json_916562499.warc index 72278284..f3f04dc6 100644 --- a/tests/test-data/__recordings__/request_1051330994/gmxhr-backend_368618529/includes-JSON-parser-in-textual-responses_3773065511.warc +++ b/tests/test-data/__recordings__/request_1051330994/json_916562499.warc @@ -1,8 +1,8 @@ WARC/1.1 -WARC-Filename: request/gmxhr backend/includes JSON parser in textual responses -WARC-Date: 2022-08-11T18:02:24.551Z +WARC-Filename: request/json +WARC-Date: 2022-08-14T10:22:20.670Z WARC-Type: warcinfo -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields Content-Length: 119 @@ -12,53 +12,58 @@ harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.551Z +WARC-Date: 2022-08-14T10:22:20.670Z WARC-Type: request -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=request WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:bb6cedb3133323a1aacedc533310ae0e9889e6c35ae1a02976d411b5e52a1fcb -Content-Length: 22 +WARC-Block-Digest: sha256:07ff13073caf8794bb9f68e87cafb66638b2222d4b44953b6d77862c5002f82d +Content-Length: 133 GET /json HTTP/1.1 +accept: */* +accept-encoding: gzip, deflate, br +connection: close +host: httpbin.org +user-agent: node-fetch WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.551Z +WARC-Date: 2022-08-14T10:22:20.670Z WARC-Type: metadata -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:c0d60fc19fbdcc1f96bd5b60bd03e1854163114e22ce907adefa1500253374ec -WARC-Block-Digest: sha256:c0d60fc19fbdcc1f96bd5b60bd03e1854163114e22ce907adefa1500253374ec -Content-Length: 348 +WARC-Payload-Digest: sha256:7b1c997fca6b9490d0fdb30cba5686f2ffea1c66c8ba044863eaea02a2b096dc +WARC-Block-Digest: sha256:7b1c997fca6b9490d0fdb30cba5686f2ffea1c66c8ba044863eaea02a2b096dc +Content-Length: 349 -harEntryId: deeba439e3d4986011f830ebdd3be4c6 +harEntryId: 934cef95e7505987bec5a0f4e6519d90 harEntryOrder: 0 cache: {} -startedDateTime: 2022-08-11T18:02:24.210Z -time: 339 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":339,"receive":0,"ssl":-1} -warcRequestHeadersSize: 43 +startedDateTime: 2022-08-14T10:22:20.130Z +time: 475 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":475,"receive":0,"ssl":-1} +warcRequestHeadersSize: 152 warcRequestCookies: [] -warcResponseHeadersSize: 254 +warcResponseHeadersSize: 208 warcResponseCookies: [] responseDecoded: false WARC/1.1 WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.551Z +WARC-Date: 2022-08-14T10:22:20.670Z WARC-Type: response -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=response WARC-Payload-Digest: sha256:bab3dfd2a6c992e4bf589eee05fc9650cc9d6988660b947a7a87b99420d108f9 -WARC-Block-Digest: sha256:c906abcac8a40e0deafd374b57b054e6cdb21bba169d97bb6cd80828dd7bb26c -Content-Length: 700 +WARC-Block-Digest: sha256:078e22dc1f009e93d154f7258ca60e2a2657ac0de196127e3ad679a17dd30214 +Content-Length: 654 HTTP/1.1 200 OK access-control-allow-credentials: true @@ -66,9 +71,8 @@ access-control-allow-origin: * connection: close content-length: 429 content-type: application/json -date: Thu, 11 Aug 2022 18:02:24 GMT +date: Sun, 14 Aug 2022 10:22:20 GMT server: gunicorn/19.9.0 -x-pollyjs-finalurl: https://httpbin.org/json { "slideshow": { @@ -94,67 +98,63 @@ x-pollyjs-finalurl: https://httpbin.org/json WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.552Z +WARC-Date: 2022-08-14T10:22:20.670Z WARC-Type: request -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=request WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:a76c54a8b1ed0588fa05c76be7f6ef5c6c8feeb857b0f9cdc5bb746f7a9fe6a5 -Content-Length: 133 +WARC-Block-Digest: sha256:bb6cedb3133323a1aacedc533310ae0e9889e6c35ae1a02976d411b5e52a1fcb +Content-Length: 22 GET /json HTTP/1.1 -accept: */* -accept-encoding: gzip, deflate, br -connection: close -user-agent: node-fetch -host: httpbin.org WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.552Z +WARC-Date: 2022-08-14T10:22:20.670Z WARC-Type: metadata -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:f6f6bb0c4d693a3b49680424d08cc63cbc5c14285c9f55bd073866339a9b8fc0 -WARC-Block-Digest: sha256:f6f6bb0c4d693a3b49680424d08cc63cbc5c14285c9f55bd073866339a9b8fc0 -Content-Length: 349 +WARC-Payload-Digest: sha256:e712150393df07f11348b088770df1f74ce0db31d4c9f35810a511373209ea29 +WARC-Block-Digest: sha256:e712150393df07f11348b088770df1f74ce0db31d4c9f35810a511373209ea29 +Content-Length: 344 -harEntryId: 934cef95e7505987bec5a0f4e6519d90 +harEntryId: deeba439e3d4986011f830ebdd3be4c6 harEntryOrder: 0 cache: {} -startedDateTime: 2022-08-11T18:02:24.212Z -time: 333 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":333,"receive":0,"ssl":-1} -warcRequestHeadersSize: 152 +startedDateTime: 2022-08-14T10:22:20.662Z +time: 7 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":7,"receive":0,"ssl":-1} +warcRequestHeadersSize: 43 warcRequestCookies: [] -warcResponseHeadersSize: 208 +warcResponseHeadersSize: 254 warcResponseCookies: [] responseDecoded: false WARC/1.1 WARC-Target-URI: https://httpbin.org/json -WARC-Date: 2022-08-11T18:02:24.551Z +WARC-Date: 2022-08-14T10:22:20.670Z WARC-Type: response -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=response WARC-Payload-Digest: sha256:bab3dfd2a6c992e4bf589eee05fc9650cc9d6988660b947a7a87b99420d108f9 -WARC-Block-Digest: sha256:396f17be56dc8e7980fcccfd6dc30189a5390927544a75e395919dcb981b3b80 -Content-Length: 654 +WARC-Block-Digest: sha256:d8e2bed98c62ce1dd2093d38da23b278017a868bb2d26c76593e0cf900add9fb +Content-Length: 700 HTTP/1.1 200 OK -date: Thu, 11 Aug 2022 18:02:24 GMT -content-type: application/json -content-length: 429 +access-control-allow-credentials: true +access-control-allow-origin: * connection: close +content-length: 429 +content-type: application/json +date: Sun, 14 Aug 2022 10:22:20 GMT server: gunicorn/19.9.0 -access-control-allow-origin: * -access-control-allow-credentials: true +x-pollyjs-finalurl: https://httpbin.org/json { "slideshow": { diff --git a/tests/test-data/__recordings__/request_1051330994/recording-observer_1153221294/exports-recorded-domains_4272767257.warc b/tests/test-data/__recordings__/request_1051330994/recording-observer_1153221294/exports-recorded-domains_4272767257.warc new file mode 100644 index 00000000..83d3b965 --- /dev/null +++ b/tests/test-data/__recordings__/request_1051330994/recording-observer_1153221294/exports-recorded-domains_4272767257.warc @@ -0,0 +1,200 @@ +WARC/1.1 +WARC-Filename: request/recording observer/exports recorded domains +WARC-Date: 2022-10-15T09:56:29.637Z +WARC-Type: warcinfo +WARC-Record-ID: +Content-Type: application/warc-fields +Content-Length: 119 + +software: warcio.js +harVersion: 1.2 +harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} + + +WARC/1.1 +WARC-Concurrent-To: +WARC-Target-URI: https://example.com/index.html +WARC-Date: 2022-10-15T09:56:29.638Z +WARC-Type: request +WARC-Record-ID: +Content-Type: application/http; msgtype=request +WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +WARC-Block-Digest: sha256:fe1741a1b062814f2da097ac360ee8d6dd6a3a7d27e2e42a1a03026ecdd1c62f +Content-Length: 28 + +GET /index.html HTTP/1.1 + + + +WARC/1.1 +WARC-Concurrent-To: +WARC-Target-URI: https://example.com/index.html +WARC-Date: 2022-10-15T09:56:29.639Z +WARC-Type: metadata +WARC-Record-ID: +Content-Type: application/warc-fields +WARC-Payload-Digest: sha256:cc66e2341a8aaec97996c6c160ef21cff25a30d1164c8654e5026f6ea4c26f9d +WARC-Block-Digest: sha256:cc66e2341a8aaec97996c6c160ef21cff25a30d1164c8654e5026f6ea4c26f9d +Content-Length: 348 + +harEntryId: d56e8038a43ca4ce986a9de3418509d6 +harEntryOrder: 0 +cache: {} +startedDateTime: 2022-10-15T09:56:29.121Z +time: 512 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":512,"receive":0,"ssl":-1} +warcRequestHeadersSize: 49 +warcRequestCookies: [] +warcResponseHeadersSize: 428 +warcResponseCookies: [] +responseDecoded: false + + +WARC/1.1 +WARC-Target-URI: https://example.com/index.html +WARC-Date: 2022-10-15T09:56:29.638Z +WARC-Type: response +WARC-Record-ID: +Content-Type: application/http; msgtype=response +WARC-Payload-Digest: sha256:ea8fac7c65fb589b0d53560f5251f74f9e9b243478dcb6b3ea79b5e36449c8d9 +WARC-Block-Digest: sha256:49cb67a2636bb264824e5d52feb9a0b6631f492c79904ba1c8304c01064ecd61 +Content-Length: 1701 + +HTTP/1.1 200 OK +accept-ranges: bytes +age: 294646 +cache-control: max-age=604800 +connection: close +content-encoding: gzip +content-length: 648 +content-type: text/html; charset=UTF-8 +date: Sat, 15 Oct 2022 09:56:29 GMT +etag: "3147526947" +expires: Sat, 22 Oct 2022 09:56:29 GMT +last-modified: Thu, 17 Oct 2019 07:18:26 GMT +server: ECS (dcb/7F83) +vary: Accept-Encoding +x-cache: HIT +x-pollyjs-finalurl: https://example.com/index.html + + + + + Example Domain + + + + + + + + + + + + + +WARC/1.1 +WARC-Concurrent-To: +WARC-Target-URI: https://example.com/index.html +WARC-Date: 2022-10-15T09:56:29.639Z +WARC-Type: request +WARC-Record-ID: +Content-Type: application/http; msgtype=request +WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +WARC-Block-Digest: sha256:4c61d59769d26e708f47e3de8165b4984a5ab0531f075e41c09345d82f821257 +Content-Length: 139 + +GET /index.html HTTP/1.1 +accept: */* +accept-encoding: gzip, deflate, br +connection: close +user-agent: node-fetch +host: example.com + + + +WARC/1.1 +WARC-Concurrent-To: +WARC-Target-URI: https://example.com/index.html +WARC-Date: 2022-10-15T09:56:29.639Z +WARC-Type: metadata +WARC-Record-ID: +Content-Type: application/warc-fields +WARC-Payload-Digest: sha256:19cdba96f862a96215914d3bc1e87ec580cea4ec985ff2707f86bcbbac61d174 +WARC-Block-Digest: sha256:19cdba96f862a96215914d3bc1e87ec580cea4ec985ff2707f86bcbbac61d174 +Content-Length: 386 + +harEntryId: 1d5bfcd970ce7605a05dd93529b6311c +harEntryOrder: 0 +cache: {} +startedDateTime: 2022-10-15T09:56:29.124Z +time: 504 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":504,"receive":0,"ssl":-1} +warcRequestHeadersSize: 158 +warcRequestCookies: [] +warcResponseHeadersSize: 376 +warcResponseCookies: [] +responseDecoded: false +warcResponseContentEncoding: base64 + + +WARC/1.1 +WARC-Target-URI: https://example.com/index.html +WARC-Date: 2022-10-15T09:56:29.639Z +WARC-Type: response +WARC-Record-ID: +Content-Type: application/http; msgtype=response +WARC-Payload-Digest: sha256:09de4599a1f79054f7e28b62b2e2d85d461fbf7abb37a7ff67804e7e959ff41f +WARC-Block-Digest: sha256:a6b56f5fb0ff34056a79f00434c1587ce4ac737438f1e51336e3e7e1e70a803c +Content-Length: 1261 + +HTTP/1.1 200 OK +content-encoding: gzip +accept-ranges: bytes +age: 294646 +cache-control: max-age=604800 +content-type: text/html; charset=UTF-8 +date: Sat, 15 Oct 2022 09:56:29 GMT +etag: "3147526947" +expires: Sat, 22 Oct 2022 09:56:29 GMT +last-modified: Thu, 17 Oct 2019 07:18:26 GMT +server: ECS (dcb/7F83) +vary: Accept-Encoding +x-cache: HIT +content-length: 648 +connection: close + +["H4sIAAAAAAAA/31UTXPbIBC9+1ds1UsyIyQnaRqPLWn6mWkPaQ9pDz0SsbKYCFAByfZ08t+7Qo4jN5makYFdeLvvsZC9Eqb0uxah9qopZtljh1wUM6Bf5qVvsPi85aptED4ZxaXO0tE6G5co9BzKmluHPo86X7FFBGkxcdbetwx/d7LPo49Ge9SeDWEjKMdZHnnc+nQIvzpAvYSkucI86iVuWmP9ZP9GCl/nAntZIguTGKSWXvKGuZI3mJ89QTm/IzJDBvvApXPR6LszYgd/wjBMeXm/tqbTgpWmMXYJr6s5tfPVYYnidi31EuZPppYLIfX6yFZRpqziSja7JTDekpzM7ZxHFcPYs07G8KGR+v6Gl7fBdE2bYohucW0Qfn6NaPy9RQ23XLth8gWbHr0sOXzDDslyMMTw3hJ3wqalzKGV1VMuYfAQ/oXsJ3SDcEt4O5+32+cML1EB77x5geg5qtV/RRPUJhncGSvQMsuF7BzplFweAZgtczUXZkPI7RYu6Luibxjb9R0/mcehJfPz09WEDF8O6sXU99JJj2JC7TGTi8WbxWKSyXD+TGBpLPfSEEttNE5B3ykUksOJ4lu21+dq0Od0An6s4lFV/KPYROVjx8MkZJaGCi3CWWXpeB1n2VCbdDsp2L6O67NnN5NMo68tftTSgQh2oFFlLHQOYZg1Tef8QLhHwBHBDQ56DjpF98kl8Mt0RGIXtnhCGqtlj6ahIXkJoLNIdHxtOg+tlRSiNHS0Ugcxgebc3VOFhOgtWiWdI0eSpe0hz4weCItVHg3PhFum6WazSSTXPDF2nY4hXbpPMypujB1IEKAKQZKE0HgRELM0iJOle6nS8UH7CyjrfG/oBAAA"] + diff --git a/tests/test-data/__recordings__/request_1051330994/status-200_1423196613.warc b/tests/test-data/__recordings__/request_1051330994/status-200_1423196613.warc new file mode 100644 index 00000000..93b531fc --- /dev/null +++ b/tests/test-data/__recordings__/request_1051330994/status-200_1423196613.warc @@ -0,0 +1,139 @@ +WARC/1.1 +WARC-Filename: request/status 200 +WARC-Date: 2022-08-14T10:22:22.202Z +WARC-Type: warcinfo +WARC-Record-ID: +Content-Type: application/warc-fields +Content-Length: 119 + +software: warcio.js +harVersion: 1.2 +harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} + + +WARC/1.1 +WARC-Concurrent-To: +WARC-Target-URI: https://httpbin.org/status/200 +WARC-Date: 2022-08-14T10:22:22.202Z +WARC-Type: request +WARC-Record-ID: +Content-Type: application/http; msgtype=request +WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +WARC-Block-Digest: sha256:9922d14b77062497aa13cdfa85c711da0999348a1e1e8ec507a8b5f3901b27b4 +Content-Length: 28 + +GET /status/200 HTTP/1.1 + + + +WARC/1.1 +WARC-Concurrent-To: +WARC-Target-URI: https://httpbin.org/status/200 +WARC-Date: 2022-08-14T10:22:22.202Z +WARC-Type: metadata +WARC-Record-ID: +Content-Type: application/warc-fields +WARC-Payload-Digest: sha256:cf0e963bdae27d6fecb364b942f2b3dc8285f29fab9d5e5dcdff75fa6475bd55 +WARC-Block-Digest: sha256:cf0e963bdae27d6fecb364b942f2b3dc8285f29fab9d5e5dcdff75fa6475bd55 +Content-Length: 350 + +harEntryId: a9fcf4b9db5edfb19e7c091c3ad17df3 +harEntryOrder: 0 +cache: {} +startedDateTime: 2022-08-14T10:22:21.170Z +time: 1030 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":1030,"receive":0,"ssl":-1} +warcRequestHeadersSize: 49 +warcRequestCookies: [] +warcResponseHeadersSize: 266 +warcResponseCookies: [] +responseDecoded: false + + +WARC/1.1 +WARC-Target-URI: https://httpbin.org/status/200 +WARC-Date: 2022-08-14T10:22:22.202Z +WARC-Type: response +WARC-Record-ID: +Content-Type: application/http; msgtype=response +WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +WARC-Block-Digest: sha256:1644b53a444b343debffe747d4691ccf4e53e30d67455612cbdc666b01e5ddda +Content-Length: 283 + +HTTP/1.1 200 OK +access-control-allow-credentials: true +access-control-allow-origin: * +connection: close +content-length: 0 +content-type: text/html; charset=utf-8 +date: Sun, 14 Aug 2022 10:22:22 GMT +server: gunicorn/19.9.0 +x-pollyjs-finalurl: https://httpbin.org/status/200 + + + +WARC/1.1 +WARC-Concurrent-To: +WARC-Target-URI: https://httpbin.org/status/200 +WARC-Date: 2022-08-14T10:22:22.203Z +WARC-Type: request +WARC-Record-ID: +Content-Type: application/http; msgtype=request +WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +WARC-Block-Digest: sha256:673b9c92623149f7d971ea051892ac71552082268c7277e37e2c0275d3cad19f +Content-Length: 139 + +GET /status/200 HTTP/1.1 +accept: */* +accept-encoding: gzip, deflate, br +connection: close +user-agent: node-fetch +host: httpbin.org + + + +WARC/1.1 +WARC-Concurrent-To: +WARC-Target-URI: https://httpbin.org/status/200 +WARC-Date: 2022-08-14T10:22:22.203Z +WARC-Type: metadata +WARC-Record-ID: +Content-Type: application/warc-fields +WARC-Payload-Digest: sha256:7a808165fd513ef191afa62c5f32cbce23ac8395710f93a7f97d5f2dfe497fc9 +WARC-Block-Digest: sha256:7a808165fd513ef191afa62c5f32cbce23ac8395710f93a7f97d5f2dfe497fc9 +Content-Length: 351 + +harEntryId: f8e66635ed118f4b7123d24dfc78d645 +harEntryOrder: 0 +cache: {} +startedDateTime: 2022-08-14T10:22:21.172Z +time: 1023 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":1023,"receive":0,"ssl":-1} +warcRequestHeadersSize: 158 +warcRequestCookies: [] +warcResponseHeadersSize: 214 +warcResponseCookies: [] +responseDecoded: false + + +WARC/1.1 +WARC-Target-URI: https://httpbin.org/status/200 +WARC-Date: 2022-08-14T10:22:22.202Z +WARC-Type: response +WARC-Record-ID: +Content-Type: application/http; msgtype=response +WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 +WARC-Block-Digest: sha256:b4371c00454f12a90649031f1ad12076720a9b30075599bab360be74249561c2 +Content-Length: 231 + +HTTP/1.1 200 OK +date: Sun, 14 Aug 2022 10:22:22 GMT +content-type: text/html; charset=utf-8 +content-length: 0 +connection: close +server: gunicorn/19.9.0 +access-control-allow-origin: * +access-control-allow-credentials: true + + + diff --git a/tests/test-data/__recordings__/request_1051330994/frontend_3860412285/rejects-on-HTTP-error-by-default_484413642.warc b/tests/test-data/__recordings__/request_1051330994/status-404_1079651855.warc similarity index 56% rename from tests/test-data/__recordings__/request_1051330994/frontend_3860412285/rejects-on-HTTP-error-by-default_484413642.warc rename to tests/test-data/__recordings__/request_1051330994/status-404_1079651855.warc index e78aaac7..9f4db814 100644 --- a/tests/test-data/__recordings__/request_1051330994/frontend_3860412285/rejects-on-HTTP-error-by-default_484413642.warc +++ b/tests/test-data/__recordings__/request_1051330994/status-404_1079651855.warc @@ -1,8 +1,8 @@ WARC/1.1 -WARC-Filename: request/frontend/rejects on HTTP error by default -WARC-Date: 2022-08-11T18:16:32.046Z +WARC-Filename: request/status 404 +WARC-Date: 2022-08-14T10:22:21.157Z WARC-Type: warcinfo -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields Content-Length: 119 @@ -12,11 +12,11 @@ harCreator: {"name":"Polly.JS","version":"6.0.5","comment":"persister:fs-warc"} WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.047Z +WARC-Date: 2022-08-14T10:22:21.157Z WARC-Type: request -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=request WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 WARC-Block-Digest: sha256:a893948eb21d3dbec7e1a032e518fb041bef4fde63aee2f2753d9e8e107b9305 @@ -27,22 +27,22 @@ GET /status/404 HTTP/1.1 WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.047Z +WARC-Date: 2022-08-14T10:22:21.157Z WARC-Type: metadata -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:b98e5d1212f13227b0a4f2625d1ad01ca83b4398ac4de9350c1066af4600dc61 -WARC-Block-Digest: sha256:b98e5d1212f13227b0a4f2625d1ad01ca83b4398ac4de9350c1066af4600dc61 +WARC-Payload-Digest: sha256:cf4b146ef49e31a21fca2bff082c8ab2c50edb29a191274383a99f9d10ba4c96 +WARC-Block-Digest: sha256:cf4b146ef49e31a21fca2bff082c8ab2c50edb29a191274383a99f9d10ba4c96 Content-Length: 348 harEntryId: 55b7d2fddaf85f12d5cd082798fdf191 harEntryOrder: 0 cache: {} -startedDateTime: 2022-08-11T18:16:31.543Z -time: 484 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":484,"receive":0,"ssl":-1} +startedDateTime: 2022-08-14T10:22:20.687Z +time: 468 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":468,"receive":0,"ssl":-1} warcRequestHeadersSize: 49 warcRequestCookies: [] warcResponseHeadersSize: 266 @@ -52,12 +52,12 @@ responseDecoded: false WARC/1.1 WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.047Z +WARC-Date: 2022-08-14T10:22:21.157Z WARC-Type: response -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=response WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:16e535a56374897c7e77c761a39eab2fff76e6b02a50f16b138de4c4899e2fc3 +WARC-Block-Digest: sha256:70db8e1ad053f90bac38c36d2b8ed52809ad5cc95bff5d705e6a78ec720c5516 Content-Length: 290 HTTP/1.1 404 Not Found @@ -66,18 +66,18 @@ access-control-allow-origin: * connection: close content-length: 0 content-type: text/html; charset=utf-8 -date: Thu, 11 Aug 2022 18:16:32 GMT +date: Sun, 14 Aug 2022 10:22:21 GMT server: gunicorn/19.9.0 x-pollyjs-finalurl: https://httpbin.org/status/404 WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.048Z +WARC-Date: 2022-08-14T10:22:21.157Z WARC-Type: request -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=request WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 WARC-Block-Digest: sha256:5157e6b9b580a6c7e9cee7f2e3a20e2f13b8cdeee951b5e7473d56e9616b38d1 @@ -93,22 +93,22 @@ host: httpbin.org WARC/1.1 -WARC-Concurrent-To: +WARC-Concurrent-To: WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.048Z +WARC-Date: 2022-08-14T10:22:21.157Z WARC-Type: metadata -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/warc-fields -WARC-Payload-Digest: sha256:1560cbb600e13206ad4c6c83e18a00ad01d83bfadbaf8e08b07cb065bd5f3141 -WARC-Block-Digest: sha256:1560cbb600e13206ad4c6c83e18a00ad01d83bfadbaf8e08b07cb065bd5f3141 +WARC-Payload-Digest: sha256:f390aa05566bd1b21af2e3a7208d33b5281331761967081c5a50286a167518bb +WARC-Block-Digest: sha256:f390aa05566bd1b21af2e3a7208d33b5281331761967081c5a50286a167518bb Content-Length: 349 harEntryId: a504acfefdddde0737bdc537a8462565 harEntryOrder: 0 cache: {} -startedDateTime: 2022-08-11T18:16:31.545Z -time: 479 -timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":479,"receive":0,"ssl":-1} +startedDateTime: 2022-08-14T10:22:20.689Z +time: 462 +timings: {"blocked":-1,"dns":-1,"connect":-1,"send":0,"wait":462,"receive":0,"ssl":-1} warcRequestHeadersSize: 158 warcRequestCookies: [] warcResponseHeadersSize: 214 @@ -118,16 +118,16 @@ responseDecoded: false WARC/1.1 WARC-Target-URI: https://httpbin.org/status/404 -WARC-Date: 2022-08-11T18:16:32.047Z +WARC-Date: 2022-08-14T10:22:21.157Z WARC-Type: response -WARC-Record-ID: +WARC-Record-ID: Content-Type: application/http; msgtype=response WARC-Payload-Digest: sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 -WARC-Block-Digest: sha256:6d4910838f2c934093777fd591eb82f0bc8b982f2a2e65e9d1ded485384f5540 +WARC-Block-Digest: sha256:7c77bff559d03958cadfff81abaec132ed0180889ea83b8cc47e9785653654fb Content-Length: 238 HTTP/1.1 404 Not Found -date: Thu, 11 Aug 2022 18:16:32 GMT +date: Sun, 14 Aug 2022 10:22:21 GMT content-type: text/html; charset=utf-8 content-length: 0 connection: close diff --git a/tests/unit/lib/logging/collectorSink.test.ts b/tests/unit/lib/logging/collectorSink.test.ts new file mode 100644 index 00000000..856968b6 --- /dev/null +++ b/tests/unit/lib/logging/collectorSink.test.ts @@ -0,0 +1,56 @@ +import { CollectorSink } from '@lib/logging/collectorSink'; +import { LogLevel } from '@lib/logging/levels'; +import { Logger } from '@lib/logging/logger'; + +describe('collector sink', () => { + let sink: CollectorSink; + let logger: Logger; + const dateNowMock = jest.spyOn(Date, 'now'); + let expectedTimestamp: string; + + beforeEach(() => { + // Need to reset sink and logger on each case so that the saved messages + // are reset + sink = new CollectorSink(); + logger = new Logger({ + logLevel: LogLevel.INFO, + sinks: [sink], + }); + dateNowMock.mockReturnValue(123); + expectedTimestamp = new Date(Date.now()).toISOString(); + }); + + it('places a timestamp on log messages', () => { + logger.info('this is a message'); + + expect(sink.dumpMessages()).toBe(`[${expectedTimestamp} - INFO] this is a message`); + }); + + it('logs multiple messages', () => { + logger.info('this is a message'); + dateNowMock.mockReturnValue(456); + logger.info('this is another message'); + const expectedTimestamp2 = new Date(Date.now()).toISOString(); + + expect(sink.dumpMessages().split('\n')).toStrictEqual([ + `[${expectedTimestamp} - INFO] this is a message`, + `[${expectedTimestamp2} - INFO] this is another message`, + ]); + }); + + it('logs debug messages', () => { + logger.debug('this is a debug message'); + + expect(sink.dumpMessages()).toBe(`[${expectedTimestamp} - DEBUG] this is a debug message`); + }); + + it('logs exception information messages', () => { + const err = new Error('hello world'); + logger.error('this is an error', err); + + expect(sink.dumpMessages().split('\n')).toStrictEqual([ + `[${expectedTimestamp} - ERROR] this is an error`, + 'Error: hello world', + ]); + }); +}); diff --git a/tests/unit/lib/logging/logger.test.ts b/tests/unit/lib/logging/logger.test.ts index c764c369..dc0194e0 100644 --- a/tests/unit/lib/logging/logger.test.ts +++ b/tests/unit/lib/logging/logger.test.ts @@ -14,11 +14,12 @@ class FakeSink implements LoggingSink { public readonly onWarn = jest.fn(); } -const handlerNames: Array = ['onDebug', 'onLog', 'onWarn', 'onError', 'onInfo', 'onSuccess']; -type LoggerMethodName = 'debug' | 'log' | 'warn' | 'error' | 'info' | 'success'; -const loggerMethodNames: LoggerMethodName[] = ['debug', 'log', 'info', 'success', 'warn', 'error']; +const loggerMethodNames = ['debug', 'log', 'info', 'success', 'warn', 'error'] as const; +const handlerNames = ['onDebug', 'onLog', 'onWarn', 'onError', 'onInfo', 'onSuccess'] as const; +type LoggerMethodName = typeof loggerMethodNames[number]; +type LogHandlerName = typeof handlerNames[number]; const loggerToHandlerNames = Object.fromEntries(loggerMethodNames - .map((name) => [name, 'on' + name[0].toUpperCase() + name.slice(1) as keyof LoggingSink])); + .map((name) => [name, 'on' + name[0].toUpperCase() + name.slice(1) as LogHandlerName])); describe('logger', () => { describe('configuring', () => { @@ -201,4 +202,17 @@ describe('logger', () => { .toHaveBeenCalledTimes(shouldCall ? 1 : 0); }); }); + + it('allows sink to override logging level', () => { + const sink = new FakeSink() as LoggingSink; + sink.minimumLevel = LogLevel.DEBUG; + const logger = new Logger({ + logLevel: LogLevel.INFO, + sinks: [sink], + }); + + logger.debug('test message'); + + expect(sink.onDebug).toHaveBeenCalledOnce(); + }); }); diff --git a/tests/unit/lib/util/request.test.ts b/tests/unit/lib/util/request.test.ts index c8b7d18f..36475c56 100644 --- a/tests/unit/lib/util/request.test.ts +++ b/tests/unit/lib/util/request.test.ts @@ -1,6 +1,8 @@ import NodeHttpAdapter from '@pollyjs/adapter-node-http'; +import { LOGGER } from '@lib/logging/logger'; import { AbortedError, HTTPResponseError, NetworkError, request, RequestBackend, TimeoutError } from '@lib/util/request'; +import { loggingObserver, RecordingObserver } from '@lib/util/request/observers'; import { mockGMxmlHttpRequest } from '@test-utils/gm_mocks'; import { mockFetch, setupPolly } from '@test-utils/pollyjs'; import GMXHRAdapter from '@test-utils/pollyjs/gmxhr-adapter'; @@ -21,6 +23,32 @@ describe('request', () => { mockFetch(); }); + beforeEach(() => { + // Set up pollyjs so we reuse the same recordings for requests that + // we're making multiple times. + const server = pollyContext.polly.server; + server + .get(httpBinHelloWorldUrl) + // FIXME: We need to keep recordings for GMXHR contentType separate. + // The recordings for arraybuffer and blob are encoded as Base64, + // and the pollyjs GMXHR adapter needs that information to return + // the correct response. We should fix the adapter to not require + // a specific encoding. If we reuse the recordings for these, the + // adapter will return a response with `responseText` even though + // the `contentType` is set to `blob` or `arraybuffer`. + .filter((req) => !req.recordingName.includes('gmxhr backend/supports')) + .recordingName('request/hello world'); + server + .get('https://httpbin.org/json') + .recordingName('request/json'); + server + .get('https://httpbin.org/status/404') + .recordingName('request/status 404'); + server + .get('https://httpbin.org/status/200') + .recordingName('request/status 200'); + }); + describe.each([['fetch', RequestBackend.FETCH], ['gmxhr', RequestBackend.GMXHR]])('%s backend', (_1, backend) => { it.each(responseTypeCases)('supports %s responseType requests', async (responseType, property, expectedValue) => { const response = await request.get(httpBinHelloWorldUrl, { @@ -186,4 +214,210 @@ describe('request', () => { expect(cb).toHaveBeenCalledWith('multiple,values', 'test2', resp.headers); }); }); + + describe('observers', () => { + const fakeObserver = { + onStarted: jest.fn(), + onFailed: jest.fn(), + onSuccess: jest.fn(), + }; + + beforeAll(() => { + request.addObserver(fakeObserver); + }); + + beforeEach(() => { + fakeObserver.onStarted.mockReset(); + fakeObserver.onFailed.mockReset(); + fakeObserver.onSuccess.mockReset(); + }); + + it('notifies onStarted', async () => { + await request.get('https://httpbin.org/status/200'); + + expect(fakeObserver.onStarted).toHaveBeenCalledOnce(); + }); + + it('notifies onSuccess on success', async () => { + await request.get('https://httpbin.org/status/200'); + + expect(fakeObserver.onSuccess).toHaveBeenCalledOnce(); + expect(fakeObserver.onFailed).not.toHaveBeenCalled(); + }); + + it('notifies onFailure on errors', async () => { + pollyContext.polly.configure({ + recordFailedRequests: true, + }); + + await expect(request.get('https://httpbin.org/status/404')).toReject(); + + expect(fakeObserver.onFailed).toHaveBeenCalledOnce(); + expect(fakeObserver.onSuccess).not.toHaveBeenCalled(); + }); + }); + + describe('logging observer', () => { + const debugSpy = jest.spyOn(LOGGER, 'debug'); + + beforeAll(() => { + // This will remain attached after these tests are finished because + // we can't remove observers at this time. It's not a problem though. + request.addObserver(loggingObserver); + }); + + beforeEach(() => { + debugSpy.mockClear(); + }); + + it('logs when request started', async () => { + await request.get('https://httpbin.org/status/200'); + + // Should've been called multiple times. + expect(debugSpy).toHaveBeenNthCalledWith(1, 'GET https://httpbin.org/status/200 - STARTED (backend: 2)'); + }); + + it('notifies onSuccess on success', async () => { + await request.get('https://httpbin.org/status/200'); + + expect(debugSpy).toHaveBeenLastCalledWith('GET https://httpbin.org/status/200 - SUCCESS (code 200)'); + }); + + it('notifies onFailure on errors', async () => { + pollyContext.polly.configure({ + recordFailedRequests: true, + }); + + await expect(request.get('https://httpbin.org/status/404')).toReject(); + + expect(debugSpy).toHaveBeenLastCalledWith('GET https://httpbin.org/status/404 - FAILED (HTTPResponseError: HTTP error 404: Not Found)'); + }); + }); + + describe('recording observer', () => { + const observer = new RecordingObserver(); + + beforeAll(() => { + request.addObserver(observer); + }); + + beforeEach(() => { + while (observer['recordedResponses'].length > 0) { + observer['recordedResponses'].pop(); + } + }); + + it('exports successful requests', async () => { + await request.get(httpBinHelloWorldUrl); + + expect(observer.hasRecordings()).toBeTrue(); + expect(observer.exportResponses()).toIncludeMultiple([ + `GET ${httpBinHelloWorldUrl} (backend: 2)`, + `${httpBinHelloWorldUrl} 200: OK`, + 'content-length: 11', + 'hello world', + ]); + }); + + it('does not include onProgress callback in exported options', async () => { + await request.get(httpBinHelloWorldUrl, { + // eslint-disable-next-line @typescript-eslint/no-empty-function + onProgress() {}, + }); + + expect(observer.hasRecordings()).toBeTrue(); + expect(observer.exportResponses()).toIncludeMultiple([ + `GET ${httpBinHelloWorldUrl} (backend: 2)`, + 'Options: {}', + `${httpBinHelloWorldUrl} 200: OK`, + 'content-length: 11', + 'hello world', + ]); + }); + + it('exports successful arraybuffer requests', async () => { + await request.get(httpBinHelloWorldUrl, { + responseType: 'arraybuffer', + backend: RequestBackend.FETCH, + }); + + expect(observer.hasRecordings()).toBeTrue(); + expect(observer.exportResponses()).toIncludeMultiple([ + `GET ${httpBinHelloWorldUrl} (backend: 1)`, + ('Options: {\n' + + ' "responseType": "arraybuffer",\n' + + ' "backend": 1\n' + + '}'), + `${httpBinHelloWorldUrl} 200:`, + 'content-length: 11', + '', + ]); + }); + + it('exports successful blob requests', async () => { + await request.get(httpBinHelloWorldUrl, { + responseType: 'blob', + backend: RequestBackend.FETCH, + }); + + expect(observer.hasRecordings()).toBeTrue(); + expect(observer.exportResponses()).toIncludeMultiple([ + `GET ${httpBinHelloWorldUrl} (backend: 1)`, + ('Options: {\n' + + ' "responseType": "blob",\n' + + ' "backend": 1\n' + + '}'), + `${httpBinHelloWorldUrl} 200:`, + 'content-length: 11', + '', // Probably internally encoded as Base64? + ]); + }); + + it('exports failed requests', async () => { + await expect(request.get('https://httpbin.org/status/404')).toReject(); + + expect(observer.hasRecordings()).toBeTrue(); + expect(observer.exportResponses()).toIncludeMultiple([ + 'GET https://httpbin.org/status/404 (backend: 2)', + 'https://httpbin.org/status/404 404: Not Found', + 'content-length: 0', + ]); + }); + + it('does not export requests that failed due to network errors', async () => { + // There's no response to export, so this shouldn't be exported at all. + mockGMxmlHttpRequest.mockImplementation((options) => options.onerror?.({} as GM.Response)); + + expect(observer.hasRecordings()).toBeFalse(); + await expect(request.get('https://httpbin.org/status/200', { + backend: RequestBackend.GMXHR, + })).toReject(); + + expect(observer.exportResponses()).toBeEmpty(); + }); + + it('exports multiple requests', async () => { + await expect(request.get('https://httpbin.org/status/200')).toResolve(); + await expect(request.get('https://httpbin.org/status/404')).toReject(); + + expect(observer.hasRecordings()).toBeTrue(); + expect(observer.exportResponses()).toIncludeMultiple([ + 'GET https://httpbin.org/status/200 (backend: 2)', + 'https://httpbin.org/status/200 200: OK', + 'content-length: 0', + '=================', + 'GET https://httpbin.org/status/404 (backend: 2)', + 'https://httpbin.org/status/404 404: Not Found', + 'content-length: 0', + ]); + }); + + it('exports recorded domains', async () => { + await expect(request.get('https://httpbin.org/status/200')).toResolve(); + await expect(request.get('https://example.com/index.html')).toResolve(); + + expect(observer.hasRecordings()).toBeTrue(); + expect(observer.recordedDomains).toIncludeSameMembers(['httpbin.org', 'example.com']); + }); + }); }); diff --git a/tests/unit/mb_enhanced_cover_art_uploads/fetch.test.ts b/tests/unit/mb_enhanced_cover_art_uploads/fetch.test.ts index 2d1b351f..88418b30 100644 --- a/tests/unit/mb_enhanced_cover_art_uploads/fetch.test.ts +++ b/tests/unit/mb_enhanced_cover_art_uploads/fetch.test.ts @@ -17,7 +17,6 @@ import { CoverArtProvider } from '@src/mb_enhanced_cover_art_uploads/providers/b import { createBlobResponse, createCoverArt, createHttpError, createImageFile } from './test-utils/dummy-data'; jest.mock('p-retry'); -jest.mock('@lib/logging/logger'); jest.mock('@lib/util/request'); // We need to provide a mock factory, because for some reason, either jest or // rewire is not recognising the generator, leading to `getMaximisedCandidates` @@ -32,8 +31,6 @@ jest.mock('@src/mb_enhanced_cover_art_uploads/form'); const mockpRetry = pRetry as jest.MockedFunction; // eslint-disable-next-line jest/unbound-method const mockRequestGet = request.get as unknown as jest.Mock, [string | URL, unknown]>; -// eslint-disable-next-line jest/unbound-method -const mockLoggerWarn = LOGGER.warn as unknown as jest.Mock; const mockGetMaximisedCandidates = getMaximisedCandidates as jest.MockedFunction; const mockGetProvider = getProvider as jest.MockedFunction; const mockGetProviderByDomain = getProviderByDomain as jest.MockedFunction; @@ -102,7 +99,6 @@ beforeEach(() => { hooks.onFetchFinished.mockClear(); hooks.onFetchProgress.mockClear(); hooks.onFetchStarted.mockClear(); - mockLoggerWarn.mockReset(); }); describe('fetching image contents', () => { @@ -288,10 +284,11 @@ describe('fetching image contents', () => { // URL if we provide it `undefined`. Object.defineProperty(response, 'url', { value: undefined }); mockRequestGet.mockResolvedValueOnce(response); + const loggerWarnSpy = jest.spyOn(LOGGER, 'warn'); await expect(fetchImageContents(new URL('https://example.com/working'), 'test.jpg', 0, {})) .toResolve(); - expect(mockLoggerWarn).toHaveBeenCalledWith(expect.stringContaining('redirect')); + expect(loggerWarnSpy).toHaveBeenCalledWith(expect.stringContaining('redirect')); }); it('assigns unique ID to each file name', async () => { diff --git a/tests/unit/mb_enhanced_cover_art_uploads/providers/base.test.ts b/tests/unit/mb_enhanced_cover_art_uploads/providers/base.test.ts index a7bc7544..a1c9d79c 100644 --- a/tests/unit/mb_enhanced_cover_art_uploads/providers/base.test.ts +++ b/tests/unit/mb_enhanced_cover_art_uploads/providers/base.test.ts @@ -12,11 +12,8 @@ import { createBlob, createBlobResponse, createFetchedImage, createTextResponse import { registerMatchers } from '../test-utils/matchers'; jest.mock('@lib/util/request'); -jest.mock('@lib/logging/logger'); // eslint-disable-next-line jest/unbound-method const mockRequestGet = request.get as unknown as jest.Mock, [string | URL, unknown]>; -// eslint-disable-next-line jest/unbound-method -const mockLoggerWarn = LOGGER.warn as unknown as jest.Mock; const findImagesMock = jest.fn(); @@ -26,7 +23,6 @@ beforeAll(() => { afterEach(() => { mockRequestGet.mockReset(); - mockLoggerWarn.mockReset(); }); describe('cover art providers', () => { @@ -176,10 +172,11 @@ describe('cover art providers', () => { ...dummyResponse, url: undefined, }); + const loggerWarnSpy = jest.spyOn(LOGGER, 'warn'); await expect(fakeProvider.fetchPage(new URL('https://example.com/test'))) .resolves.toBe('1234'); - expect(mockLoggerWarn).toHaveBeenCalledWith(expect.stringContaining('redirect')); + expect(loggerWarnSpy).toHaveBeenCalledWith(expect.stringContaining('redirect')); }); });