From 344fbd7f3316a819275bd18b766e32ec46313c01 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 3 Mar 2025 10:12:57 +0100 Subject: [PATCH 1/4] handle imported files --- .../push/__snapshots__/plugin.test.ts.snap | 11 ++ __tests__/push/plugin.test.ts | 127 +++++++++++++++++- package-lock.json | 10 ++ package.json | 1 + src/core/transform.ts | 35 ++++- src/loader.ts | 6 +- src/push/bundler.ts | 47 ++++++- tsconfig.json | 2 +- 8 files changed, 225 insertions(+), 14 deletions(-) diff --git a/__tests__/push/__snapshots__/plugin.test.ts.snap b/__tests__/push/__snapshots__/plugin.test.ts.snap index ed2a97c4..7dab8519 100644 --- a/__tests__/push/__snapshots__/plugin.test.ts.snap +++ b/__tests__/push/__snapshots__/plugin.test.ts.snap @@ -1,5 +1,16 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`Asset Plugin should include imported assets in the bundle 1`] = ` +"(() => { + // __tests__/push/test-assets/sample.pdf + var sample_default = 'export default "/Users/shahzad/elastic/synthetics/__tests__/push/test-assets/sample.pdf";'; + + // __tests__/push/test-assets/bundle.journey.ts + console.log("PDF file path:", sample_default); +})(); +" +`; + exports[`SyntheticsBundlePlugin skip locally resolved synthetics package 1`] = ` "var import__ = require("../../"); (0, import__.journey)("journey 1", () => { diff --git a/__tests__/push/plugin.test.ts b/__tests__/push/plugin.test.ts index 4f794c41..7ceb4b12 100644 --- a/__tests__/push/plugin.test.ts +++ b/__tests__/push/plugin.test.ts @@ -24,10 +24,12 @@ */ import * as esbuild from 'esbuild'; -import { mkdir, rm, writeFile } from 'fs/promises'; +import { mkdir, readFile, rm, writeFile } from 'fs/promises'; import { join } from 'path'; +import AdmZip from 'adm-zip'; import { commonOptions } from '../../src/core/transform'; import { SyntheticsBundlePlugin } from '../../src/push/plugin'; +import { Bundler } from '../../src/push/bundler'; describe('SyntheticsBundlePlugin', () => { const PROJECT_DIR = join(__dirname, 'test-bundler'); @@ -62,3 +64,126 @@ journey('journey 1', () => { expect(result.outputFiles[0].text).toMatchSnapshot(); }); }); + +describe('Asset Plugin', () => { + const PROJECT_DIR = join(__dirname, 'test-assets'); + const journeyFile = join(PROJECT_DIR, 'bundle.journey.ts'); + const assetFile = join(PROJECT_DIR, 'sample.pdf'); + const zipOutput = join(PROJECT_DIR, 'output.zip'); + const csvFile = join(PROJECT_DIR, 'sample.csv'); + const assetContent = 'This is a test PDF file'; + + beforeAll(async () => { + await mkdir(PROJECT_DIR, { recursive: true }); + + // Create a sample asset file + await writeFile(assetFile, assetContent); + + // Create a journey file that imports the asset + await writeFile( + journeyFile, + `import pdfPath from './sample.pdf'; +console.log("PDF file path:", pdfPath);` + ); + }); + + afterAll(async () => { + await rm(PROJECT_DIR, { recursive: true, force: true }); + }); + + it('should include imported assets in the bundle', async () => { + const assets: { [key: string]: Uint8Array } = {}; + + const assetPlugin: esbuild.Plugin = { + name: 'asset-plugin', + setup(build) { + build.onLoad({ filter: /\.(pdf|xlsx?|csv)$/ }, async args => { + assets[args.path] = await readFile(args.path); + return { + contents: `export default ${JSON.stringify(args.path)};`, // Keep path reference + loader: 'text', + }; + }); + }, + }; + + const result = await esbuild.build({ + bundle: true, + sourcemap: false, + write: false, + entryPoints: [journeyFile], + plugins: [assetPlugin], + }); + + expect(result.outputFiles).toBeDefined(); + expect(Object.keys(assets)).toContain(assetFile); + + // Ensure the asset is referenced in the output + const bundleText = result.outputFiles[0].text; + expect(bundleText).toMatchSnapshot(); + }); + + it('should bundle, zip, and contain the correct pdf asset content', async () => { + const bundler = new Bundler(); + // Build & zip + const base64Zip = await bundler.build(journeyFile, zipOutput); + await mkdir(PROJECT_DIR + '/zip/', { recursive: true }); + + const zipPath = join(PROJECT_DIR, '/zip/test-output.zip'); + + // Convert Base64 back to a ZIP file + await writeFile(zipPath, Buffer.from(base64Zip, 'base64')); + + // Read the generated ZIP file + const zip = new AdmZip(zipPath); + const zipEntries = zip.getEntries().map(entry => entry.entryName); + + // Check that the asset is included in the ZIP archive + expect(zipEntries).toEqual([ + '__tests__/push/test-assets/bundle.journey.ts', + '__tests__/push/test-assets/sample.pdf', + ]); + + // Extract and verify asset content + const extractedAsset = zip.readAsText( + '__tests__/push/test-assets/sample.pdf' + ); + expect(extractedAsset).toBe(assetContent); + }); + + it('should bundle, zip, and contain the correct csv asset content', async () => { + await writeFile( + journeyFile, + ` + import sampleCsv from './sample.csv'; + console.log("CSV file path:", sampleCsv); + ` + ); + await writeFile(csvFile, `id,name\n1,Test User\n2,Another User`); + const bundler = new Bundler(); + // Build & zip + const base64Zip = await bundler.build(journeyFile, zipOutput); + await mkdir(PROJECT_DIR + '/zip/', { recursive: true }); + + const zipPath = join(PROJECT_DIR, '/zip/test-output.zip'); + + // Convert Base64 back to a ZIP file + await writeFile(zipPath, Buffer.from(base64Zip, 'base64')); + + // Read the generated ZIP file + const zip = new AdmZip(zipPath); + const zipEntries = zip.getEntries().map(entry => entry.entryName); + + // Check that the asset is included in the ZIP archive + expect(zipEntries).toEqual([ + '__tests__/push/test-assets/bundle.journey.ts', + '__tests__/push/test-assets/sample.csv', + ]); + + // Extract and verify asset content + const extractedAsset = zip.readAsText( + '__tests__/push/test-assets/sample.csv' + ); + expect(extractedAsset).toBe(`id,name\n1,Test User\n2,Another User`); + }); +}); diff --git a/package-lock.json b/package-lock.json index cabbfd49..7111145c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -49,6 +49,7 @@ "@types/stack-utils": "^2.0.1", "@typescript-eslint/eslint-plugin": "^5.38.0", "@typescript-eslint/parser": "^5.38.0", + "adm-zip": "^0.5.16", "eslint": "^8.23.1", "husky": "^4.3.6", "is-positive": "3.1.0", @@ -4638,6 +4639,15 @@ "node": ">=0.4.0" } }, + "node_modules/adm-zip": { + "version": "0.5.16", + "resolved": "https://registry.npmjs.org/adm-zip/-/adm-zip-0.5.16.tgz", + "integrity": "sha512-TGw5yVi4saajsSEgz25grObGHEUaDrniwvA2qwSC060KfqGPdglhvPMA2lPIoxs3PQIItj2iag35fONcQqgUaQ==", + "dev": true, + "engines": { + "node": ">=12.0" + } + }, "node_modules/agent-base": { "version": "7.1.3", "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.3.tgz", diff --git a/package.json b/package.json index 0aaf2828..b983522c 100644 --- a/package.json +++ b/package.json @@ -90,6 +90,7 @@ "@types/stack-utils": "^2.0.1", "@typescript-eslint/eslint-plugin": "^5.38.0", "@typescript-eslint/parser": "^5.38.0", + "adm-zip": "^0.5.16", "eslint": "^8.23.1", "husky": "^4.3.6", "is-positive": "3.1.0", diff --git a/src/core/transform.ts b/src/core/transform.ts index c1ce6c30..4ca65aeb 100644 --- a/src/core/transform.ts +++ b/src/core/transform.ts @@ -58,6 +58,32 @@ sourceMapSupport.install({ }, }); +const TEXT_EXTS = [ + '.csv', + '.pdf', + '.docx', + '.odt', + '.xlsx', + '.png', + '.jpg', + '.jpeg', + '.gif', + '.webp', + '.zip', + '.tar.gz', + '.rar', + '.7z', + '.txt', + '.log', + '.json', + '.xml', + '.mp3', + '.wav', + '.mp4', + '.avi', + '.webm', +]; + /** * Default list of files and corresponding loaders we support * while pushing project based monitors @@ -71,6 +97,7 @@ const LOADERS: Record = { const getLoader = (filename: string) => { const ext = path.extname(filename); + if (TEXT_EXTS.includes(ext)) return 'text'; return LOADERS[ext] || 'default'; }; @@ -92,7 +119,7 @@ export function commonOptions(): CommonOptions { /** * Transform the given code using esbuild and save the corresponding - * map file in memory to be retrived later. + * map file in memory to be retried later. */ export function transform( code: string, @@ -105,7 +132,7 @@ export function transform( loader: getLoader(filename), /** * Add this only for the transformation phase, using it on - * bundling phase would disable tree shaking and uncessary bloat + * bundling phase would disable tree shaking and unnecessary bloat * * Ensures backwards compatability with tsc's implicit strict behaviour */ @@ -148,7 +175,9 @@ export function installTransform() { const { code } = transform(source, filename); return code; }, - { exts: ['.ts', '.js', '.mjs', '.cjs'] } + { + exts: ['.ts', '.js', '.mjs', '.cjs', ...TEXT_EXTS], + } // List of file extensions to hook ); return () => { diff --git a/src/loader.ts b/src/loader.ts index 9ef367b0..e6b71d64 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -80,7 +80,7 @@ export async function loadTestFiles(options: CliArgs, args: string[]) { const files = args.length > 0 ? args : (await readStdin()).split('\n').filter(Boolean); const suites = await prepareSuites(files, options.pattern); - requireSuites(suites); + await requireSuites(suites); } const loadInlineScript = source => { @@ -122,9 +122,9 @@ async function readStdin() { return chunks.join(); } -function requireSuites(suites: Iterable) { +async function requireSuites(suites: Iterable) { for (const suite of suites) { - require(suite); + await import(suite); // Use a correct relative path for each suite } } diff --git a/src/push/bundler.ts b/src/push/bundler.ts index 6f009d96..693114ba 100644 --- a/src/push/bundler.ts +++ b/src/push/bundler.ts @@ -24,7 +24,7 @@ */ import path from 'path'; -import { unlink, readFile } from 'fs/promises'; +import { readFile, unlink } from 'fs/promises'; import { createWriteStream } from 'fs'; import * as esbuild from 'esbuild'; import archiver from 'archiver'; @@ -37,6 +37,26 @@ function relativeToCwd(entry: string) { export class Bundler { async bundle(absPath: string) { + const assets: { [key: string]: Uint8Array } = {}; // Store asset files + const assetPlugin: esbuild.Plugin = { + name: 'asset-plugin', + setup(build) { + build.onLoad( + { + filter: + /\.(pdf|docx?|odt|csv|xlsx?|png|jpe?g|gif|webp|zip|tar\.gz|rar|7z|txt|log|json|xml|mp3|wav|mp4|avi|webm)$/, + }, + async args => { + const content = await readFile(args.path, 'utf-8'); // Read file contents as text + assets[args.path] = await readFile(args.path); + return { + contents: `export default ${JSON.stringify(content)};`, + loader: 'text', + }; + } + ); + }, + }; const options: esbuild.BuildOptions = { ...commonOptions(), ...{ @@ -46,17 +66,22 @@ export class Bundler { minifyWhitespace: true, sourcemap: 'inline', external: ['@elastic/synthetics'], - plugins: [SyntheticsBundlePlugin()], + plugins: [SyntheticsBundlePlugin(), assetPlugin], }, }; const result = await esbuild.build(options); if (result.errors.length > 0) { throw result.errors; } - return result.outputFiles[0].text; + return { code: result.outputFiles[0].text, assets }; } - async zip(source: string, code: string, dest: string) { + async zip( + source: string, + code: string, + dest: string, + assets: { [key: string]: Uint8Array } + ) { return new Promise((fulfill, reject) => { const output = createWriteStream(dest); const archive = archiver('zip', { @@ -72,13 +97,23 @@ export class Bundler { name: relativePath, date: new Date('1970-01-01'), }); + + // Add asset files + for (const [filePath, content] of Object.entries(assets)) { + const assetRelativePath = relativeToCwd(filePath); + archive.append(content, { + name: assetRelativePath, + date: new Date('1970-01-01'), + }); + } + archive.finalize(); }); } async build(entry: string, output: string) { - const code = await this.bundle(entry); - await this.zip(entry, code, output); + const { code, assets } = await this.bundle(entry); + await this.zip(entry, code, output, assets); const content = await this.encode(output); await this.cleanup(output); return content; diff --git a/tsconfig.json b/tsconfig.json index 436f00f6..2b6c23a2 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,5 +12,5 @@ "skipLibCheck": true, "moduleResolution": "Node" }, - "include": ["src/**/*"] + "include": ["src/**/*", "src/types/*.d.ts"] } From b25d99a71f1fd90b99a2eaa69b3bfff80698f4eb Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 3 Mar 2025 10:25:02 +0100 Subject: [PATCH 2/4] update test --- __tests__/push/__snapshots__/plugin.test.ts.snap | 11 ----------- __tests__/push/plugin.test.ts | 2 +- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/__tests__/push/__snapshots__/plugin.test.ts.snap b/__tests__/push/__snapshots__/plugin.test.ts.snap index 7dab8519..ed2a97c4 100644 --- a/__tests__/push/__snapshots__/plugin.test.ts.snap +++ b/__tests__/push/__snapshots__/plugin.test.ts.snap @@ -1,16 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Asset Plugin should include imported assets in the bundle 1`] = ` -"(() => { - // __tests__/push/test-assets/sample.pdf - var sample_default = 'export default "/Users/shahzad/elastic/synthetics/__tests__/push/test-assets/sample.pdf";'; - - // __tests__/push/test-assets/bundle.journey.ts - console.log("PDF file path:", sample_default); -})(); -" -`; - exports[`SyntheticsBundlePlugin skip locally resolved synthetics package 1`] = ` "var import__ = require("../../"); (0, import__.journey)("journey 1", () => { diff --git a/__tests__/push/plugin.test.ts b/__tests__/push/plugin.test.ts index 7ceb4b12..b8a29a16 100644 --- a/__tests__/push/plugin.test.ts +++ b/__tests__/push/plugin.test.ts @@ -120,7 +120,7 @@ console.log("PDF file path:", pdfPath);` // Ensure the asset is referenced in the output const bundleText = result.outputFiles[0].text; - expect(bundleText).toMatchSnapshot(); + expect(bundleText).toContain('push/test-assets/sample.pdf'); }); it('should bundle, zip, and contain the correct pdf asset content', async () => { From bb2731f5bb97d054fbf80fcd673d7d471766e7d0 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 3 Mar 2025 10:28:07 +0100 Subject: [PATCH 3/4] revert --- src/loader.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/loader.ts b/src/loader.ts index e6b71d64..7b8b81f4 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -80,7 +80,7 @@ export async function loadTestFiles(options: CliArgs, args: string[]) { const files = args.length > 0 ? args : (await readStdin()).split('\n').filter(Boolean); const suites = await prepareSuites(files, options.pattern); - await requireSuites(suites); + requireSuites(suites); } const loadInlineScript = source => { @@ -122,9 +122,9 @@ async function readStdin() { return chunks.join(); } -async function requireSuites(suites: Iterable) { +function requireSuites(suites: Iterable) { for (const suite of suites) { - await import(suite); // Use a correct relative path for each suite + require(suite); // Use a correct relative path for each suite } } From 730416554c881b25fd5e6815e7e0e78cdec1d538 Mon Sep 17 00:00:00 2001 From: Shahzad Date: Mon, 3 Mar 2025 10:28:47 +0100 Subject: [PATCH 4/4] revert --- src/loader.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/loader.ts b/src/loader.ts index 7b8b81f4..9ef367b0 100644 --- a/src/loader.ts +++ b/src/loader.ts @@ -124,7 +124,7 @@ async function readStdin() { function requireSuites(suites: Iterable) { for (const suite of suites) { - require(suite); // Use a correct relative path for each suite + require(suite); } }